2024-11-27 16:48:30,701 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-27 16:48:30,715 main DEBUG Took 0.012041 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-27 16:48:30,715 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-27 16:48:30,716 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-27 16:48:30,717 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-27 16:48:30,718 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 16:48:30,726 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-27 16:48:30,740 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 16:48:30,742 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 16:48:30,743 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 16:48:30,744 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 16:48:30,744 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 16:48:30,745 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 16:48:30,746 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 16:48:30,746 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 16:48:30,747 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 16:48:30,747 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 16:48:30,748 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 16:48:30,749 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 16:48:30,749 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 16:48:30,749 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 16:48:30,750 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 16:48:30,751 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 16:48:30,751 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 16:48:30,751 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 16:48:30,752 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 16:48:30,752 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 16:48:30,753 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 16:48:30,753 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 16:48:30,753 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 16:48:30,753 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 16:48:30,754 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 16:48:30,754 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-27 16:48:30,755 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 16:48:30,757 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-27 16:48:30,758 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-27 16:48:30,759 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-27 16:48:30,760 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-27 16:48:30,760 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-27 16:48:30,769 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-27 16:48:30,772 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-27 16:48:30,774 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-27 16:48:30,774 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-27 16:48:30,774 main DEBUG createAppenders(={Console}) 2024-11-27 16:48:30,775 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-27 16:48:30,775 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-27 16:48:30,776 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-27 16:48:30,776 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-27 16:48:30,777 main DEBUG OutputStream closed 2024-11-27 16:48:30,777 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-27 16:48:30,777 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-27 16:48:30,777 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-27 16:48:30,858 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-27 16:48:30,861 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-27 16:48:30,862 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-27 16:48:30,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-27 16:48:30,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-27 16:48:30,864 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-27 16:48:30,864 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-27 16:48:30,865 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-27 16:48:30,865 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-27 16:48:30,866 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-27 16:48:30,866 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-27 16:48:30,867 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-27 16:48:30,867 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-27 16:48:30,868 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-27 16:48:30,868 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-27 16:48:30,868 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-27 16:48:30,869 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-27 16:48:30,870 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-27 16:48:30,873 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27 16:48:30,873 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-27 16:48:30,874 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-27 16:48:30,875 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-27T16:48:31,198 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652 2024-11-27 16:48:31,201 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-27 16:48:31,202 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27T16:48:31,214 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-27T16:48:31,222 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayValueCompression timeout: 13 mins 2024-11-27T16:48:31,268 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-27T16:48:31,334 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-27T16:48:31,335 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-27T16:48:31,353 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-27T16:48:31,375 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197, deleteOnExit=true 2024-11-27T16:48:31,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-27T16:48:31,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/test.cache.data in system properties and HBase conf 2024-11-27T16:48:31,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/hadoop.tmp.dir in system properties and HBase conf 2024-11-27T16:48:31,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/hadoop.log.dir in system properties and HBase conf 2024-11-27T16:48:31,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-27T16:48:31,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-27T16:48:31,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-27T16:48:31,486 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-27T16:48:31,616 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T16:48:31,621 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-27T16:48:31,621 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-27T16:48:31,622 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-27T16:48:31,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T16:48:31,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-27T16:48:31,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-27T16:48:31,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T16:48:31,625 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T16:48:31,626 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-27T16:48:31,626 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/nfs.dump.dir in system properties and HBase conf 2024-11-27T16:48:31,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/java.io.tmpdir in system properties and HBase conf 2024-11-27T16:48:31,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T16:48:31,628 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-27T16:48:31,628 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-27T16:48:32,526 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-27T16:48:32,640 INFO [Time-limited test {}] log.Log(170): Logging initialized @2787ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-27T16:48:32,739 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T16:48:32,814 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T16:48:32,854 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T16:48:32,854 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T16:48:32,856 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T16:48:32,873 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T16:48:32,877 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/hadoop.log.dir/,AVAILABLE} 2024-11-27T16:48:32,878 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T16:48:33,112 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3717288f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/java.io.tmpdir/jetty-localhost-40505-hadoop-hdfs-3_4_1-tests_jar-_-any-554973451658592403/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T16:48:33,119 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:40505} 2024-11-27T16:48:33,119 INFO [Time-limited test {}] server.Server(415): Started @3267ms 2024-11-27T16:48:33,542 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T16:48:33,549 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T16:48:33,550 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T16:48:33,551 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T16:48:33,551 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T16:48:33,552 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@433df981{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/hadoop.log.dir/,AVAILABLE} 2024-11-27T16:48:33,553 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cf5a85e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T16:48:33,681 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@38011c53{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/java.io.tmpdir/jetty-localhost-43827-hadoop-hdfs-3_4_1-tests_jar-_-any-16747416598142040549/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T16:48:33,682 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@63f56910{HTTP/1.1, (http/1.1)}{localhost:43827} 2024-11-27T16:48:33,682 INFO [Time-limited test {}] server.Server(415): Started @3830ms 2024-11-27T16:48:33,738 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T16:48:33,881 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T16:48:33,888 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T16:48:33,893 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T16:48:33,893 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T16:48:33,893 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T16:48:33,895 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ebbf344{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/hadoop.log.dir/,AVAILABLE} 2024-11-27T16:48:33,896 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46b092e1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T16:48:34,048 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5a400978{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/java.io.tmpdir/jetty-localhost-39359-hadoop-hdfs-3_4_1-tests_jar-_-any-6644792508116941025/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T16:48:34,049 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13a44756{HTTP/1.1, (http/1.1)}{localhost:39359} 2024-11-27T16:48:34,049 INFO [Time-limited test {}] server.Server(415): Started @4197ms 2024-11-27T16:48:34,052 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T16:48:34,103 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T16:48:34,107 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T16:48:34,112 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T16:48:34,112 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T16:48:34,112 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T16:48:34,113 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f079a76{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/hadoop.log.dir/,AVAILABLE} 2024-11-27T16:48:34,114 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a6d5e13{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T16:48:34,225 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/data/data4/current/BP-89784467-172.17.0.2-1732726112257/current, will proceed with Du for space computation calculation, 2024-11-27T16:48:34,225 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/data/data1/current/BP-89784467-172.17.0.2-1732726112257/current, will proceed with Du for space computation calculation, 2024-11-27T16:48:34,225 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/data/data3/current/BP-89784467-172.17.0.2-1732726112257/current, will proceed with Du for space computation calculation, 2024-11-27T16:48:34,225 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/data/data2/current/BP-89784467-172.17.0.2-1732726112257/current, will proceed with Du for space computation calculation, 2024-11-27T16:48:34,259 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7822234{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/java.io.tmpdir/jetty-localhost-38653-hadoop-hdfs-3_4_1-tests_jar-_-any-13815382002493087125/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T16:48:34,261 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8f81303{HTTP/1.1, (http/1.1)}{localhost:38653} 2024-11-27T16:48:34,261 INFO [Time-limited test {}] server.Server(415): Started @4408ms 2024-11-27T16:48:34,263 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T16:48:34,298 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T16:48:34,299 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T16:48:34,385 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x53646a2823ffd12b with lease ID 0x50c64d5bffdce7b8: Processing first storage report for DS-ed211189-b496-4df4-badd-0783b484a6d0 from datanode DatanodeRegistration(127.0.0.1:37511, datanodeUuid=c746d6c3-7b4e-4aa7-83e7-ac27b2137949, infoPort=39655, infoSecurePort=0, ipcPort=43979, storageInfo=lv=-57;cid=testClusterID;nsid=812180456;c=1732726112257) 2024-11-27T16:48:34,386 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x53646a2823ffd12b with lease ID 0x50c64d5bffdce7b8: from storage DS-ed211189-b496-4df4-badd-0783b484a6d0 node DatanodeRegistration(127.0.0.1:37511, datanodeUuid=c746d6c3-7b4e-4aa7-83e7-ac27b2137949, infoPort=39655, infoSecurePort=0, ipcPort=43979, storageInfo=lv=-57;cid=testClusterID;nsid=812180456;c=1732726112257), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-27T16:48:34,387 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4d9e590a5dc9a9ef with lease ID 0x50c64d5bffdce7b9: Processing first storage report for DS-a471d440-70d4-449b-9f22-dafdb2116171 from datanode DatanodeRegistration(127.0.0.1:41739, datanodeUuid=bcc98693-a8fe-41ef-8eb8-7e87ebc1854e, infoPort=33829, infoSecurePort=0, ipcPort=38561, storageInfo=lv=-57;cid=testClusterID;nsid=812180456;c=1732726112257) 2024-11-27T16:48:34,387 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4d9e590a5dc9a9ef with lease ID 0x50c64d5bffdce7b9: from storage DS-a471d440-70d4-449b-9f22-dafdb2116171 node DatanodeRegistration(127.0.0.1:41739, datanodeUuid=bcc98693-a8fe-41ef-8eb8-7e87ebc1854e, infoPort=33829, infoSecurePort=0, ipcPort=38561, storageInfo=lv=-57;cid=testClusterID;nsid=812180456;c=1732726112257), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T16:48:34,387 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x53646a2823ffd12b with lease ID 0x50c64d5bffdce7b8: Processing first storage report for DS-55b76b7c-5976-4b9d-8dc3-106f52457e46 from datanode DatanodeRegistration(127.0.0.1:37511, datanodeUuid=c746d6c3-7b4e-4aa7-83e7-ac27b2137949, infoPort=39655, infoSecurePort=0, ipcPort=43979, storageInfo=lv=-57;cid=testClusterID;nsid=812180456;c=1732726112257) 2024-11-27T16:48:34,388 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x53646a2823ffd12b with lease ID 0x50c64d5bffdce7b8: from storage DS-55b76b7c-5976-4b9d-8dc3-106f52457e46 node DatanodeRegistration(127.0.0.1:37511, datanodeUuid=c746d6c3-7b4e-4aa7-83e7-ac27b2137949, infoPort=39655, infoSecurePort=0, ipcPort=43979, storageInfo=lv=-57;cid=testClusterID;nsid=812180456;c=1732726112257), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T16:48:34,388 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4d9e590a5dc9a9ef with lease ID 0x50c64d5bffdce7b9: Processing first storage report for DS-4dd98135-ec53-4dad-b3a0-12146a1a65d6 from datanode DatanodeRegistration(127.0.0.1:41739, datanodeUuid=bcc98693-a8fe-41ef-8eb8-7e87ebc1854e, infoPort=33829, infoSecurePort=0, ipcPort=38561, storageInfo=lv=-57;cid=testClusterID;nsid=812180456;c=1732726112257) 2024-11-27T16:48:34,388 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4d9e590a5dc9a9ef with lease ID 0x50c64d5bffdce7b9: from storage DS-4dd98135-ec53-4dad-b3a0-12146a1a65d6 node DatanodeRegistration(127.0.0.1:41739, datanodeUuid=bcc98693-a8fe-41ef-8eb8-7e87ebc1854e, infoPort=33829, infoSecurePort=0, ipcPort=38561, storageInfo=lv=-57;cid=testClusterID;nsid=812180456;c=1732726112257), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T16:48:34,402 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/data/data5/current/BP-89784467-172.17.0.2-1732726112257/current, will proceed with Du for space computation calculation, 2024-11-27T16:48:34,403 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/data/data6/current/BP-89784467-172.17.0.2-1732726112257/current, will proceed with Du for space computation calculation, 2024-11-27T16:48:34,440 WARN [Thread-123 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T16:48:34,466 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc6887212022ecb57 with lease ID 0x50c64d5bffdce7ba: Processing first storage report for DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7 from datanode DatanodeRegistration(127.0.0.1:37433, datanodeUuid=16368b57-f612-4fd5-a59b-d4eec26c19ed, infoPort=42883, infoSecurePort=0, ipcPort=44395, storageInfo=lv=-57;cid=testClusterID;nsid=812180456;c=1732726112257) 2024-11-27T16:48:34,466 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc6887212022ecb57 with lease ID 0x50c64d5bffdce7ba: from storage DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7 node DatanodeRegistration(127.0.0.1:37433, datanodeUuid=16368b57-f612-4fd5-a59b-d4eec26c19ed, infoPort=42883, infoSecurePort=0, ipcPort=44395, storageInfo=lv=-57;cid=testClusterID;nsid=812180456;c=1732726112257), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T16:48:34,466 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc6887212022ecb57 with lease ID 0x50c64d5bffdce7ba: Processing first storage report for DS-8d121adf-fff5-4914-9dc6-eecec59960e0 from datanode DatanodeRegistration(127.0.0.1:37433, datanodeUuid=16368b57-f612-4fd5-a59b-d4eec26c19ed, infoPort=42883, infoSecurePort=0, ipcPort=44395, storageInfo=lv=-57;cid=testClusterID;nsid=812180456;c=1732726112257) 2024-11-27T16:48:34,467 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc6887212022ecb57 with lease ID 0x50c64d5bffdce7ba: from storage DS-8d121adf-fff5-4914-9dc6-eecec59960e0 node DatanodeRegistration(127.0.0.1:37433, datanodeUuid=16368b57-f612-4fd5-a59b-d4eec26c19ed, infoPort=42883, infoSecurePort=0, ipcPort=44395, storageInfo=lv=-57;cid=testClusterID;nsid=812180456;c=1732726112257), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T16:48:34,696 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652 2024-11-27T16:48:34,825 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/zookeeper_0, clientPort=60075, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T16:48:34,838 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60075 2024-11-27T16:48:34,854 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T16:48:34,857 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T16:48:35,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741825_1001 (size=7) 2024-11-27T16:48:35,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741825_1001 (size=7) 2024-11-27T16:48:35,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741825_1001 (size=7) 2024-11-27T16:48:35,495 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4 with version=8 2024-11-27T16:48:35,496 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/hbase-staging 2024-11-27T16:48:35,819 INFO [Time-limited test {}] client.ConnectionUtils(128): master/37bfedfd9766:0 server-side Connection retries=45 2024-11-27T16:48:35,829 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T16:48:35,830 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T16:48:35,834 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T16:48:35,835 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T16:48:35,835 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T16:48:35,981 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T16:48:36,048 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-27T16:48:36,061 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-27T16:48:36,066 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T16:48:36,098 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 22425 (auto-detected) 2024-11-27T16:48:36,099 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-27T16:48:36,121 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35959 2024-11-27T16:48:36,143 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35959 connecting to ZooKeeper ensemble=127.0.0.1:60075 2024-11-27T16:48:36,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:359590x0, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T16:48:36,178 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35959-0x10039e202d30000 connected 2024-11-27T16:48:36,204 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T16:48:36,207 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T16:48:36,219 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T16:48:36,225 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4, hbase.cluster.distributed=false 2024-11-27T16:48:36,254 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T16:48:36,261 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35959 2024-11-27T16:48:36,261 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35959 2024-11-27T16:48:36,262 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35959 2024-11-27T16:48:36,264 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35959 2024-11-27T16:48:36,265 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35959 2024-11-27T16:48:36,382 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/37bfedfd9766:0 server-side Connection retries=45 2024-11-27T16:48:36,384 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T16:48:36,384 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T16:48:36,384 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T16:48:36,385 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T16:48:36,385 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T16:48:36,388 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T16:48:36,390 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T16:48:36,391 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40989 2024-11-27T16:48:36,393 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40989 connecting to ZooKeeper ensemble=127.0.0.1:60075 2024-11-27T16:48:36,394 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T16:48:36,397 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T16:48:36,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:409890x0, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T16:48:36,403 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40989-0x10039e202d30001 connected 2024-11-27T16:48:36,403 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T16:48:36,408 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T16:48:36,416 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T16:48:36,419 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T16:48:36,424 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T16:48:36,425 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40989 2024-11-27T16:48:36,426 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40989 2024-11-27T16:48:36,431 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40989 2024-11-27T16:48:36,432 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40989 2024-11-27T16:48:36,432 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40989 2024-11-27T16:48:36,451 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/37bfedfd9766:0 server-side Connection retries=45 2024-11-27T16:48:36,451 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T16:48:36,451 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T16:48:36,452 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T16:48:36,452 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T16:48:36,452 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T16:48:36,452 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T16:48:36,453 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T16:48:36,454 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37003 2024-11-27T16:48:36,456 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37003 connecting to ZooKeeper ensemble=127.0.0.1:60075 2024-11-27T16:48:36,457 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T16:48:36,459 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T16:48:36,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:370030x0, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T16:48:36,466 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:370030x0, quorum=127.0.0.1:60075, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T16:48:36,466 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37003-0x10039e202d30002 connected 2024-11-27T16:48:36,466 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T16:48:36,467 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T16:48:36,468 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37003-0x10039e202d30002, quorum=127.0.0.1:60075, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T16:48:36,470 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37003-0x10039e202d30002, quorum=127.0.0.1:60075, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T16:48:36,471 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37003 2024-11-27T16:48:36,471 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37003 2024-11-27T16:48:36,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37003 2024-11-27T16:48:36,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37003 2024-11-27T16:48:36,473 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37003 2024-11-27T16:48:36,492 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/37bfedfd9766:0 server-side Connection retries=45 2024-11-27T16:48:36,492 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T16:48:36,493 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T16:48:36,493 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T16:48:36,493 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T16:48:36,493 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T16:48:36,493 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T16:48:36,494 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T16:48:36,495 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36103 2024-11-27T16:48:36,497 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36103 connecting to ZooKeeper ensemble=127.0.0.1:60075 2024-11-27T16:48:36,499 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T16:48:36,502 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T16:48:36,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:361030x0, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T16:48:36,510 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:361030x0, quorum=127.0.0.1:60075, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T16:48:36,510 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36103-0x10039e202d30003 connected 2024-11-27T16:48:36,511 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T16:48:36,514 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T16:48:36,516 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T16:48:36,518 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T16:48:36,520 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36103 2024-11-27T16:48:36,520 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36103 2024-11-27T16:48:36,521 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36103 2024-11-27T16:48:36,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36103 2024-11-27T16:48:36,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36103 2024-11-27T16:48:36,539 DEBUG [M:0;37bfedfd9766:35959 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;37bfedfd9766:35959 2024-11-27T16:48:36,541 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/37bfedfd9766,35959,1732726115618 2024-11-27T16:48:36,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37003-0x10039e202d30002, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T16:48:36,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T16:48:36,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T16:48:36,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T16:48:36,550 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/37bfedfd9766,35959,1732726115618 2024-11-27T16:48:36,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T16:48:36,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:36,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:36,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T16:48:36,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:36,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37003-0x10039e202d30002, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T16:48:36,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37003-0x10039e202d30002, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:36,574 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T16:48:36,575 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/37bfedfd9766,35959,1732726115618 from backup master directory 2024-11-27T16:48:36,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37003-0x10039e202d30002, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T16:48:36,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/37bfedfd9766,35959,1732726115618 2024-11-27T16:48:36,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T16:48:36,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T16:48:36,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T16:48:36,579 WARN [master/37bfedfd9766:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T16:48:36,579 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=37bfedfd9766,35959,1732726115618 2024-11-27T16:48:36,582 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-27T16:48:36,584 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-27T16:48:36,642 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/hbase.id] with ID: 7a7dfc1d-b607-4ef1-a912-89ad055552df 2024-11-27T16:48:36,642 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/.tmp/hbase.id 2024-11-27T16:48:36,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741826_1002 (size=42) 2024-11-27T16:48:36,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741826_1002 (size=42) 2024-11-27T16:48:36,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741826_1002 (size=42) 2024-11-27T16:48:36,658 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/.tmp/hbase.id]:[hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/hbase.id] 2024-11-27T16:48:36,704 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T16:48:36,709 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T16:48:36,729 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-27T16:48:36,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:36,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:36,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37003-0x10039e202d30002, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:36,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:36,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741827_1003 (size=196) 2024-11-27T16:48:36,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741827_1003 (size=196) 2024-11-27T16:48:36,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741827_1003 (size=196) 2024-11-27T16:48:36,770 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T16:48:36,773 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T16:48:36,780 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:36,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741828_1004 (size=1189) 2024-11-27T16:48:36,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741828_1004 (size=1189) 2024-11-27T16:48:36,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741828_1004 (size=1189) 2024-11-27T16:48:36,837 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store 2024-11-27T16:48:36,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741829_1005 (size=34) 2024-11-27T16:48:36,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741829_1005 (size=34) 2024-11-27T16:48:36,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741829_1005 (size=34) 2024-11-27T16:48:36,872 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-27T16:48:36,876 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:36,878 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T16:48:36,878 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T16:48:36,879 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T16:48:36,881 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T16:48:36,881 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T16:48:36,881 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T16:48:36,883 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732726116878Disabling compacts and flushes for region at 1732726116878Disabling writes for close at 1732726116881 (+3 ms)Writing region close event to WAL at 1732726116881Closed at 1732726116881 2024-11-27T16:48:36,885 WARN [master/37bfedfd9766:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store/.initializing 2024-11-27T16:48:36,885 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/WALs/37bfedfd9766,35959,1732726115618 2024-11-27T16:48:36,893 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T16:48:36,910 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=37bfedfd9766%2C35959%2C1732726115618, suffix=, logDir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/WALs/37bfedfd9766,35959,1732726115618, archiveDir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/oldWALs, maxLogs=10 2024-11-27T16:48:36,951 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/WALs/37bfedfd9766,35959,1732726115618/37bfedfd9766%2C35959%2C1732726115618.1732726116916, exclude list is [], retry=0 2024-11-27T16:48:36,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:48:36,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:36,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:36,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:36,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-27T16:48:37,023 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/WALs/37bfedfd9766,35959,1732726115618/37bfedfd9766%2C35959%2C1732726115618.1732726116916 2024-11-27T16:48:37,024 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:39655:39655)] 2024-11-27T16:48:37,024 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:48:37,025 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:37,028 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T16:48:37,029 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T16:48:37,068 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T16:48:37,098 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T16:48:37,102 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:37,106 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T16:48:37,106 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T16:48:37,110 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T16:48:37,110 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:37,111 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:37,111 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T16:48:37,114 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T16:48:37,114 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:37,115 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:37,115 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T16:48:37,117 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T16:48:37,118 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:37,118 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:37,119 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T16:48:37,122 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T16:48:37,123 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T16:48:37,128 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T16:48:37,129 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T16:48:37,132 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T16:48:37,135 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T16:48:37,140 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T16:48:37,142 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59032783, jitterRate=-0.12034298479557037}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T16:48:37,148 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732726117042Initializing all the Stores at 1732726117044 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732726117045 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726117045Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726117046 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726117046Cleaning up temporary data from old regions at 1732726117129 (+83 ms)Region opened successfully at 1732726117147 (+18 ms) 2024-11-27T16:48:37,149 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T16:48:37,188 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17a16cce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=37bfedfd9766/172.17.0.2:0 2024-11-27T16:48:37,221 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T16:48:37,233 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-27T16:48:37,233 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-27T16:48:37,236 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T16:48:37,238 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-27T16:48:37,243 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-27T16:48:37,243 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T16:48:37,276 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T16:48:37,288 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T16:48:37,291 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T16:48:37,294 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T16:48:37,296 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T16:48:37,298 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T16:48:37,301 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T16:48:37,306 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T16:48:37,307 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T16:48:37,309 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T16:48:37,311 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T16:48:37,329 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T16:48:37,331 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T16:48:37,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37003-0x10039e202d30002, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T16:48:37,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T16:48:37,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T16:48:37,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T16:48:37,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:37,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37003-0x10039e202d30002, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:37,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:37,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:37,340 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=37bfedfd9766,35959,1732726115618, sessionid=0x10039e202d30000, setting cluster-up flag (Was=false) 2024-11-27T16:48:37,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:37,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:37,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37003-0x10039e202d30002, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:37,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:37,360 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T16:48:37,362 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=37bfedfd9766,35959,1732726115618 2024-11-27T16:48:37,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:37,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37003-0x10039e202d30002, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:37,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:37,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:37,375 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T16:48:37,376 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=37bfedfd9766,35959,1732726115618 2024-11-27T16:48:37,383 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T16:48:37,429 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer(746): ClusterId : 7a7dfc1d-b607-4ef1-a912-89ad055552df 2024-11-27T16:48:37,429 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.HRegionServer(746): ClusterId : 7a7dfc1d-b607-4ef1-a912-89ad055552df 2024-11-27T16:48:37,429 INFO [RS:2;37bfedfd9766:36103 {}] regionserver.HRegionServer(746): ClusterId : 7a7dfc1d-b607-4ef1-a912-89ad055552df 2024-11-27T16:48:37,433 DEBUG [RS:1;37bfedfd9766:37003 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T16:48:37,433 DEBUG [RS:2;37bfedfd9766:36103 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T16:48:37,433 DEBUG [RS:0;37bfedfd9766:40989 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T16:48:37,439 DEBUG [RS:1;37bfedfd9766:37003 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T16:48:37,439 DEBUG [RS:2;37bfedfd9766:36103 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T16:48:37,439 DEBUG [RS:0;37bfedfd9766:40989 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T16:48:37,439 DEBUG [RS:1;37bfedfd9766:37003 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T16:48:37,439 DEBUG [RS:2;37bfedfd9766:36103 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T16:48:37,439 DEBUG [RS:0;37bfedfd9766:40989 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T16:48:37,445 DEBUG [RS:1;37bfedfd9766:37003 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T16:48:37,445 DEBUG [RS:0;37bfedfd9766:40989 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T16:48:37,446 DEBUG [RS:2;37bfedfd9766:36103 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T16:48:37,446 DEBUG [RS:1;37bfedfd9766:37003 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2113b0e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=37bfedfd9766/172.17.0.2:0 2024-11-27T16:48:37,446 DEBUG [RS:0;37bfedfd9766:40989 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@670a7035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=37bfedfd9766/172.17.0.2:0 2024-11-27T16:48:37,446 DEBUG [RS:2;37bfedfd9766:36103 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63f9fc7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=37bfedfd9766/172.17.0.2:0 2024-11-27T16:48:37,466 DEBUG [RS:1;37bfedfd9766:37003 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;37bfedfd9766:37003 2024-11-27T16:48:37,469 DEBUG [RS:0;37bfedfd9766:40989 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;37bfedfd9766:40989 2024-11-27T16:48:37,470 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T16:48:37,470 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T16:48:37,470 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T16:48:37,470 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T16:48:37,470 DEBUG [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T16:48:37,470 DEBUG [RS:1;37bfedfd9766:37003 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T16:48:37,474 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer(2659): reportForDuty to master=37bfedfd9766,35959,1732726115618 with port=40989, startcode=1732726116343 2024-11-27T16:48:37,474 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.HRegionServer(2659): reportForDuty to master=37bfedfd9766,35959,1732726115618 with port=37003, startcode=1732726116450 2024-11-27T16:48:37,479 DEBUG [RS:2;37bfedfd9766:36103 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;37bfedfd9766:36103 2024-11-27T16:48:37,479 INFO [RS:2;37bfedfd9766:36103 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T16:48:37,479 INFO [RS:2;37bfedfd9766:36103 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T16:48:37,479 DEBUG [RS:2;37bfedfd9766:36103 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T16:48:37,481 INFO [RS:2;37bfedfd9766:36103 {}] regionserver.HRegionServer(2659): reportForDuty to master=37bfedfd9766,35959,1732726115618 with port=36103, startcode=1732726116491 2024-11-27T16:48:37,485 INFO [AsyncFSWAL-0-hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData-prefix:37bfedfd9766,35959,1732726115618 {}] compress.Compression(560): Loaded codec org.apache.hadoop.hbase.io.compress.ReusableStreamGzipCodec for compression algorithm GZ 2024-11-27T16:48:37,488 DEBUG [RS:1;37bfedfd9766:37003 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T16:48:37,488 DEBUG [RS:0;37bfedfd9766:40989 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T16:48:37,488 DEBUG [RS:2;37bfedfd9766:36103 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T16:48:37,508 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T16:48:37,522 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T16:48:37,532 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T16:48:37,539 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 37bfedfd9766,35959,1732726115618 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T16:48:37,545 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39955, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T16:48:37,545 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56101, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T16:48:37,545 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51223, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T16:48:37,553 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35959 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T16:48:37,555 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/37bfedfd9766:0, corePoolSize=5, maxPoolSize=5 2024-11-27T16:48:37,555 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/37bfedfd9766:0, corePoolSize=5, maxPoolSize=5 2024-11-27T16:48:37,556 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/37bfedfd9766:0, corePoolSize=5, maxPoolSize=5 2024-11-27T16:48:37,556 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/37bfedfd9766:0, corePoolSize=5, maxPoolSize=5 2024-11-27T16:48:37,556 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/37bfedfd9766:0, corePoolSize=10, maxPoolSize=10 2024-11-27T16:48:37,556 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,556 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/37bfedfd9766:0, corePoolSize=2, maxPoolSize=2 2024-11-27T16:48:37,556 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,560 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732726147560 2024-11-27T16:48:37,562 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T16:48:37,563 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T16:48:37,565 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35959 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T16:48:37,566 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35959 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-27T16:48:37,568 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T16:48:37,568 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T16:48:37,568 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T16:48:37,569 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T16:48:37,569 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T16:48:37,569 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T16:48:37,572 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,577 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:37,577 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T16:48:37,581 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T16:48:37,582 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T16:48:37,583 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T16:48:37,588 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T16:48:37,588 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T16:48:37,590 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/37bfedfd9766:0:becomeActiveMaster-HFileCleaner.large.0-1732726117590,5,FailOnTimeoutGroup] 2024-11-27T16:48:37,591 DEBUG [RS:2;37bfedfd9766:36103 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-27T16:48:37,591 WARN [RS:2;37bfedfd9766:36103 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-27T16:48:37,591 DEBUG [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-27T16:48:37,591 WARN [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-27T16:48:37,591 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/37bfedfd9766:0:becomeActiveMaster-HFileCleaner.small.0-1732726117591,5,FailOnTimeoutGroup] 2024-11-27T16:48:37,591 DEBUG [RS:1;37bfedfd9766:37003 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-27T16:48:37,591 WARN [RS:1;37bfedfd9766:37003 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-27T16:48:37,591 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,592 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T16:48:37,613 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,614 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741831_1007 (size=1321) 2024-11-27T16:48:37,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741831_1007 (size=1321) 2024-11-27T16:48:37,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741831_1007 (size=1321) 2024-11-27T16:48:37,622 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T16:48:37,622 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4 2024-11-27T16:48:37,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741832_1008 (size=32) 2024-11-27T16:48:37,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741832_1008 (size=32) 2024-11-27T16:48:37,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741832_1008 (size=32) 2024-11-27T16:48:37,641 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:37,643 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T16:48:37,645 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T16:48:37,645 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:37,646 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T16:48:37,646 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T16:48:37,649 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T16:48:37,649 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:37,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T16:48:37,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T16:48:37,652 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T16:48:37,652 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:37,653 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T16:48:37,654 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T16:48:37,656 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T16:48:37,656 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:37,657 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T16:48:37,657 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T16:48:37,658 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740 2024-11-27T16:48:37,659 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740 2024-11-27T16:48:37,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T16:48:37,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T16:48:37,663 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T16:48:37,666 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T16:48:37,674 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T16:48:37,675 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73321869, jitterRate=0.09258098900318146}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T16:48:37,677 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732726117641Initializing all the Stores at 1732726117642 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732726117642Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732726117643 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726117643Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732726117643Cleaning up temporary data from old regions at 1732726117662 (+19 ms)Region opened successfully at 1732726117677 (+15 ms) 2024-11-27T16:48:37,678 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T16:48:37,678 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T16:48:37,678 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T16:48:37,678 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T16:48:37,678 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T16:48:37,680 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T16:48:37,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732726117677Disabling compacts and flushes for region at 1732726117677Disabling writes for close at 1732726117678 (+1 ms)Writing region close event to WAL at 1732726117679 (+1 ms)Closed at 1732726117680 (+1 ms) 2024-11-27T16:48:37,685 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T16:48:37,685 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T16:48:37,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T16:48:37,692 INFO [RS:2;37bfedfd9766:36103 {}] regionserver.HRegionServer(2659): reportForDuty to master=37bfedfd9766,35959,1732726115618 with port=36103, startcode=1732726116491 2024-11-27T16:48:37,692 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer(2659): reportForDuty to master=37bfedfd9766,35959,1732726115618 with port=40989, startcode=1732726116343 2024-11-27T16:48:37,693 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.HRegionServer(2659): reportForDuty to master=37bfedfd9766,35959,1732726115618 with port=37003, startcode=1732726116450 2024-11-27T16:48:37,694 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35959 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 37bfedfd9766,40989,1732726116343 2024-11-27T16:48:37,698 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35959 {}] master.ServerManager(517): Registering regionserver=37bfedfd9766,40989,1732726116343 2024-11-27T16:48:37,703 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T16:48:37,707 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T16:48:37,707 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35959 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 37bfedfd9766,37003,1732726116450 2024-11-27T16:48:37,707 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35959 {}] master.ServerManager(517): Registering regionserver=37bfedfd9766,37003,1732726116450 2024-11-27T16:48:37,708 DEBUG [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4 2024-11-27T16:48:37,708 DEBUG [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40409 2024-11-27T16:48:37,708 DEBUG [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T16:48:37,710 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35959 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 37bfedfd9766,36103,1732726116491 2024-11-27T16:48:37,710 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35959 {}] master.ServerManager(517): Registering regionserver=37bfedfd9766,36103,1732726116491 2024-11-27T16:48:37,710 DEBUG [RS:1;37bfedfd9766:37003 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4 2024-11-27T16:48:37,711 DEBUG [RS:1;37bfedfd9766:37003 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40409 2024-11-27T16:48:37,711 DEBUG [RS:1;37bfedfd9766:37003 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T16:48:37,713 DEBUG [RS:2;37bfedfd9766:36103 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4 2024-11-27T16:48:37,713 DEBUG [RS:2;37bfedfd9766:36103 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40409 2024-11-27T16:48:37,713 DEBUG [RS:2;37bfedfd9766:36103 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T16:48:37,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T16:48:37,718 DEBUG [RS:0;37bfedfd9766:40989 {}] zookeeper.ZKUtil(111): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/37bfedfd9766,40989,1732726116343 2024-11-27T16:48:37,718 DEBUG [RS:1;37bfedfd9766:37003 {}] zookeeper.ZKUtil(111): regionserver:37003-0x10039e202d30002, quorum=127.0.0.1:60075, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/37bfedfd9766,37003,1732726116450 2024-11-27T16:48:37,718 WARN [RS:0;37bfedfd9766:40989 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T16:48:37,718 WARN [RS:1;37bfedfd9766:37003 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T16:48:37,718 INFO [RS:1;37bfedfd9766:37003 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:37,718 DEBUG [RS:1;37bfedfd9766:37003 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450 2024-11-27T16:48:37,718 INFO [RS:0;37bfedfd9766:40989 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:37,719 DEBUG [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,40989,1732726116343 2024-11-27T16:48:37,719 DEBUG [RS:2;37bfedfd9766:36103 {}] zookeeper.ZKUtil(111): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/37bfedfd9766,36103,1732726116491 2024-11-27T16:48:37,719 WARN [RS:2;37bfedfd9766:36103 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T16:48:37,719 INFO [RS:2;37bfedfd9766:36103 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:37,719 DEBUG [RS:2;37bfedfd9766:36103 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,36103,1732726116491 2024-11-27T16:48:37,720 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [37bfedfd9766,36103,1732726116491] 2024-11-27T16:48:37,720 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [37bfedfd9766,40989,1732726116343] 2024-11-27T16:48:37,720 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [37bfedfd9766,37003,1732726116450] 2024-11-27T16:48:37,755 INFO [RS:2;37bfedfd9766:36103 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T16:48:37,755 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T16:48:37,755 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T16:48:37,769 INFO [RS:2;37bfedfd9766:36103 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T16:48:37,769 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T16:48:37,769 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T16:48:37,775 INFO [RS:2;37bfedfd9766:36103 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T16:48:37,775 INFO [RS:0;37bfedfd9766:40989 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T16:48:37,775 INFO [RS:1;37bfedfd9766:37003 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T16:48:37,775 INFO [RS:2;37bfedfd9766:36103 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,776 INFO [RS:0;37bfedfd9766:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,776 INFO [RS:1;37bfedfd9766:37003 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,777 INFO [RS:2;37bfedfd9766:36103 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T16:48:37,777 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T16:48:37,777 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T16:48:37,783 INFO [RS:2;37bfedfd9766:36103 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T16:48:37,783 INFO [RS:1;37bfedfd9766:37003 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T16:48:37,783 INFO [RS:0;37bfedfd9766:40989 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T16:48:37,785 INFO [RS:2;37bfedfd9766:36103 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,785 INFO [RS:0;37bfedfd9766:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,785 INFO [RS:1;37bfedfd9766:37003 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,785 DEBUG [RS:2;37bfedfd9766:36103 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,785 DEBUG [RS:1;37bfedfd9766:37003 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,785 DEBUG [RS:0;37bfedfd9766:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,785 DEBUG [RS:2;37bfedfd9766:36103 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,785 DEBUG [RS:0;37bfedfd9766:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,785 DEBUG [RS:1;37bfedfd9766:37003 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,785 DEBUG [RS:2;37bfedfd9766:36103 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,785 DEBUG [RS:0;37bfedfd9766:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,786 DEBUG [RS:2;37bfedfd9766:36103 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,786 DEBUG [RS:1;37bfedfd9766:37003 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,786 DEBUG [RS:0;37bfedfd9766:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,786 DEBUG [RS:2;37bfedfd9766:36103 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,786 DEBUG [RS:1;37bfedfd9766:37003 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,786 DEBUG [RS:2;37bfedfd9766:36103 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/37bfedfd9766:0, corePoolSize=2, maxPoolSize=2 2024-11-27T16:48:37,786 DEBUG [RS:0;37bfedfd9766:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,786 DEBUG [RS:2;37bfedfd9766:36103 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,786 DEBUG [RS:1;37bfedfd9766:37003 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,786 DEBUG [RS:0;37bfedfd9766:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/37bfedfd9766:0, corePoolSize=2, maxPoolSize=2 2024-11-27T16:48:37,786 DEBUG [RS:2;37bfedfd9766:36103 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,786 DEBUG [RS:1;37bfedfd9766:37003 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/37bfedfd9766:0, corePoolSize=2, maxPoolSize=2 2024-11-27T16:48:37,786 DEBUG [RS:0;37bfedfd9766:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,786 DEBUG [RS:2;37bfedfd9766:36103 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,786 DEBUG [RS:0;37bfedfd9766:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,786 DEBUG [RS:2;37bfedfd9766:36103 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,786 DEBUG [RS:1;37bfedfd9766:37003 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,786 DEBUG [RS:0;37bfedfd9766:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,786 DEBUG [RS:2;37bfedfd9766:36103 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,786 DEBUG [RS:1;37bfedfd9766:37003 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,787 DEBUG [RS:0;37bfedfd9766:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,787 DEBUG [RS:2;37bfedfd9766:36103 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,787 DEBUG [RS:1;37bfedfd9766:37003 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,787 DEBUG [RS:0;37bfedfd9766:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,787 DEBUG [RS:2;37bfedfd9766:36103 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/37bfedfd9766:0, corePoolSize=3, maxPoolSize=3 2024-11-27T16:48:37,787 DEBUG [RS:2;37bfedfd9766:36103 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/37bfedfd9766:0, corePoolSize=3, maxPoolSize=3 2024-11-27T16:48:37,787 DEBUG [RS:0;37bfedfd9766:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,787 DEBUG [RS:1;37bfedfd9766:37003 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,787 DEBUG [RS:0;37bfedfd9766:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/37bfedfd9766:0, corePoolSize=3, maxPoolSize=3 2024-11-27T16:48:37,787 DEBUG [RS:1;37bfedfd9766:37003 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,787 DEBUG [RS:0;37bfedfd9766:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/37bfedfd9766:0, corePoolSize=3, maxPoolSize=3 2024-11-27T16:48:37,787 DEBUG [RS:1;37bfedfd9766:37003 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/37bfedfd9766:0, corePoolSize=1, maxPoolSize=1 2024-11-27T16:48:37,787 DEBUG [RS:1;37bfedfd9766:37003 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/37bfedfd9766:0, corePoolSize=3, maxPoolSize=3 2024-11-27T16:48:37,788 DEBUG [RS:1;37bfedfd9766:37003 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/37bfedfd9766:0, corePoolSize=3, maxPoolSize=3 2024-11-27T16:48:37,788 INFO [RS:2;37bfedfd9766:36103 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,788 INFO [RS:2;37bfedfd9766:36103 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,788 INFO [RS:2;37bfedfd9766:36103 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,788 INFO [RS:2;37bfedfd9766:36103 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,788 INFO [RS:0;37bfedfd9766:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,788 INFO [RS:2;37bfedfd9766:36103 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,791 INFO [RS:2;37bfedfd9766:36103 {}] hbase.ChoreService(168): Chore ScheduledChore name=37bfedfd9766,36103,1732726116491-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T16:48:37,791 INFO [RS:0;37bfedfd9766:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,792 INFO [RS:0;37bfedfd9766:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,792 INFO [RS:0;37bfedfd9766:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,792 INFO [RS:0;37bfedfd9766:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,792 INFO [RS:0;37bfedfd9766:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=37bfedfd9766,40989,1732726116343-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T16:48:37,796 INFO [RS:1;37bfedfd9766:37003 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,796 INFO [RS:1;37bfedfd9766:37003 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,796 INFO [RS:1;37bfedfd9766:37003 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,796 INFO [RS:1;37bfedfd9766:37003 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,796 INFO [RS:1;37bfedfd9766:37003 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,797 INFO [RS:1;37bfedfd9766:37003 {}] hbase.ChoreService(168): Chore ScheduledChore name=37bfedfd9766,37003,1732726116450-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T16:48:37,813 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T16:48:37,815 INFO [RS:0;37bfedfd9766:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=37bfedfd9766,40989,1732726116343-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,815 INFO [RS:0;37bfedfd9766:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,816 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.Replication(171): 37bfedfd9766,40989,1732726116343 started 2024-11-27T16:48:37,816 INFO [RS:2;37bfedfd9766:36103 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T16:48:37,817 INFO [RS:2;37bfedfd9766:36103 {}] hbase.ChoreService(168): Chore ScheduledChore name=37bfedfd9766,36103,1732726116491-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,817 INFO [RS:2;37bfedfd9766:36103 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,817 INFO [RS:2;37bfedfd9766:36103 {}] regionserver.Replication(171): 37bfedfd9766,36103,1732726116491 started 2024-11-27T16:48:37,819 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T16:48:37,819 INFO [RS:1;37bfedfd9766:37003 {}] hbase.ChoreService(168): Chore ScheduledChore name=37bfedfd9766,37003,1732726116450-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,819 INFO [RS:1;37bfedfd9766:37003 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,819 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.Replication(171): 37bfedfd9766,37003,1732726116450 started 2024-11-27T16:48:37,835 INFO [RS:0;37bfedfd9766:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,836 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer(1482): Serving as 37bfedfd9766,40989,1732726116343, RpcServer on 37bfedfd9766/172.17.0.2:40989, sessionid=0x10039e202d30001 2024-11-27T16:48:37,836 DEBUG [RS:0;37bfedfd9766:40989 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T16:48:37,836 DEBUG [RS:0;37bfedfd9766:40989 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 37bfedfd9766,40989,1732726116343 2024-11-27T16:48:37,837 DEBUG [RS:0;37bfedfd9766:40989 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '37bfedfd9766,40989,1732726116343' 2024-11-27T16:48:37,837 DEBUG [RS:0;37bfedfd9766:40989 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T16:48:37,838 DEBUG [RS:0;37bfedfd9766:40989 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T16:48:37,839 DEBUG [RS:0;37bfedfd9766:40989 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T16:48:37,839 DEBUG [RS:0;37bfedfd9766:40989 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T16:48:37,839 DEBUG [RS:0;37bfedfd9766:40989 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 37bfedfd9766,40989,1732726116343 2024-11-27T16:48:37,839 DEBUG [RS:0;37bfedfd9766:40989 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '37bfedfd9766,40989,1732726116343' 2024-11-27T16:48:37,839 DEBUG [RS:0;37bfedfd9766:40989 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T16:48:37,840 DEBUG [RS:0;37bfedfd9766:40989 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T16:48:37,841 DEBUG [RS:0;37bfedfd9766:40989 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T16:48:37,841 INFO [RS:0;37bfedfd9766:40989 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T16:48:37,841 INFO [RS:0;37bfedfd9766:40989 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T16:48:37,841 INFO [RS:2;37bfedfd9766:36103 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,842 INFO [RS:2;37bfedfd9766:36103 {}] regionserver.HRegionServer(1482): Serving as 37bfedfd9766,36103,1732726116491, RpcServer on 37bfedfd9766/172.17.0.2:36103, sessionid=0x10039e202d30003 2024-11-27T16:48:37,842 DEBUG [RS:2;37bfedfd9766:36103 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T16:48:37,842 DEBUG [RS:2;37bfedfd9766:36103 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 37bfedfd9766,36103,1732726116491 2024-11-27T16:48:37,842 DEBUG [RS:2;37bfedfd9766:36103 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '37bfedfd9766,36103,1732726116491' 2024-11-27T16:48:37,842 DEBUG [RS:2;37bfedfd9766:36103 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T16:48:37,843 DEBUG [RS:2;37bfedfd9766:36103 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T16:48:37,843 INFO [RS:1;37bfedfd9766:37003 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:37,844 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.HRegionServer(1482): Serving as 37bfedfd9766,37003,1732726116450, RpcServer on 37bfedfd9766/172.17.0.2:37003, sessionid=0x10039e202d30002 2024-11-27T16:48:37,844 DEBUG [RS:2;37bfedfd9766:36103 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T16:48:37,844 DEBUG [RS:2;37bfedfd9766:36103 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T16:48:37,844 DEBUG [RS:1;37bfedfd9766:37003 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T16:48:37,844 DEBUG [RS:1;37bfedfd9766:37003 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 37bfedfd9766,37003,1732726116450 2024-11-27T16:48:37,844 DEBUG [RS:2;37bfedfd9766:36103 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 37bfedfd9766,36103,1732726116491 2024-11-27T16:48:37,844 DEBUG [RS:1;37bfedfd9766:37003 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '37bfedfd9766,37003,1732726116450' 2024-11-27T16:48:37,844 DEBUG [RS:1;37bfedfd9766:37003 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T16:48:37,844 DEBUG [RS:2;37bfedfd9766:36103 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '37bfedfd9766,36103,1732726116491' 2024-11-27T16:48:37,844 DEBUG [RS:2;37bfedfd9766:36103 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T16:48:37,845 DEBUG [RS:1;37bfedfd9766:37003 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T16:48:37,845 DEBUG [RS:2;37bfedfd9766:36103 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T16:48:37,846 DEBUG [RS:1;37bfedfd9766:37003 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T16:48:37,846 DEBUG [RS:1;37bfedfd9766:37003 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T16:48:37,846 DEBUG [RS:2;37bfedfd9766:36103 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T16:48:37,846 DEBUG [RS:1;37bfedfd9766:37003 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 37bfedfd9766,37003,1732726116450 2024-11-27T16:48:37,846 DEBUG [RS:1;37bfedfd9766:37003 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '37bfedfd9766,37003,1732726116450' 2024-11-27T16:48:37,846 INFO [RS:2;37bfedfd9766:36103 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T16:48:37,846 DEBUG [RS:1;37bfedfd9766:37003 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T16:48:37,846 INFO [RS:2;37bfedfd9766:36103 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T16:48:37,847 DEBUG [RS:1;37bfedfd9766:37003 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T16:48:37,848 DEBUG [RS:1;37bfedfd9766:37003 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T16:48:37,848 INFO [RS:1;37bfedfd9766:37003 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T16:48:37,848 INFO [RS:1;37bfedfd9766:37003 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T16:48:37,857 WARN [37bfedfd9766:35959 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T16:48:37,946 INFO [RS:0;37bfedfd9766:40989 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T16:48:37,947 INFO [RS:2;37bfedfd9766:36103 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T16:48:37,949 INFO [RS:1;37bfedfd9766:37003 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T16:48:37,950 INFO [RS:0;37bfedfd9766:40989 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=37bfedfd9766%2C40989%2C1732726116343, suffix=, logDir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,40989,1732726116343, archiveDir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/oldWALs, maxLogs=32 2024-11-27T16:48:37,950 INFO [RS:2;37bfedfd9766:36103 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=37bfedfd9766%2C36103%2C1732726116491, suffix=, logDir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,36103,1732726116491, archiveDir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/oldWALs, maxLogs=32 2024-11-27T16:48:37,951 INFO [RS:1;37bfedfd9766:37003 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=37bfedfd9766%2C37003%2C1732726116450, suffix=, logDir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450, archiveDir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/oldWALs, maxLogs=32 2024-11-27T16:48:37,978 DEBUG [RS:0;37bfedfd9766:40989 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,40989,1732726116343/37bfedfd9766%2C40989%2C1732726116343.1732726117955, exclude list is [], retry=0 2024-11-27T16:48:37,978 DEBUG [RS:1;37bfedfd9766:37003 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450/37bfedfd9766%2C37003%2C1732726116450.1732726117955, exclude list is [], retry=0 2024-11-27T16:48:37,978 DEBUG [RS:2;37bfedfd9766:36103 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,36103,1732726116491/37bfedfd9766%2C36103%2C1732726116491.1732726117955, exclude list is [], retry=0 2024-11-27T16:48:37,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:37,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:37,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:37,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:37,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:37,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:37,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:37,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:37,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:38,003 INFO [RS:1;37bfedfd9766:37003 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450/37bfedfd9766%2C37003%2C1732726116450.1732726117955 2024-11-27T16:48:38,005 INFO [RS:0;37bfedfd9766:40989 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,40989,1732726116343/37bfedfd9766%2C40989%2C1732726116343.1732726117955 2024-11-27T16:48:38,010 DEBUG [RS:1;37bfedfd9766:37003 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:39655:39655)] 2024-11-27T16:48:38,010 DEBUG [RS:0;37bfedfd9766:40989 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:42883:42883)] 2024-11-27T16:48:38,011 INFO [RS:2;37bfedfd9766:36103 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,36103,1732726116491/37bfedfd9766%2C36103%2C1732726116491.1732726117955 2024-11-27T16:48:38,012 DEBUG [RS:2;37bfedfd9766:36103 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:33829:33829)] 2024-11-27T16:48:38,110 DEBUG [37bfedfd9766:35959 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-27T16:48:38,122 DEBUG [37bfedfd9766:35959 {}] balancer.BalancerClusterState(204): Hosts are {37bfedfd9766=0} racks are {/default-rack=0} 2024-11-27T16:48:38,131 DEBUG [37bfedfd9766:35959 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T16:48:38,131 DEBUG [37bfedfd9766:35959 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T16:48:38,131 DEBUG [37bfedfd9766:35959 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-27T16:48:38,131 DEBUG [37bfedfd9766:35959 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T16:48:38,131 DEBUG [37bfedfd9766:35959 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T16:48:38,131 DEBUG [37bfedfd9766:35959 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-27T16:48:38,131 INFO [37bfedfd9766:35959 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T16:48:38,131 INFO [37bfedfd9766:35959 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T16:48:38,131 INFO [37bfedfd9766:35959 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-27T16:48:38,131 DEBUG [37bfedfd9766:35959 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T16:48:38,142 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=37bfedfd9766,40989,1732726116343 2024-11-27T16:48:38,147 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 37bfedfd9766,40989,1732726116343, state=OPENING 2024-11-27T16:48:38,153 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T16:48:38,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37003-0x10039e202d30002, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:38,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:38,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:38,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:48:38,156 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T16:48:38,156 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T16:48:38,156 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T16:48:38,157 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T16:48:38,159 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T16:48:38,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=37bfedfd9766,40989,1732726116343}] 2024-11-27T16:48:38,338 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T16:48:38,341 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54637, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T16:48:38,352 INFO [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T16:48:38,353 INFO [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:38,354 INFO [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T16:48:38,357 INFO [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=37bfedfd9766%2C40989%2C1732726116343.meta, suffix=.meta, logDir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,40989,1732726116343, archiveDir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/oldWALs, maxLogs=32 2024-11-27T16:48:38,375 DEBUG [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,40989,1732726116343/37bfedfd9766%2C40989%2C1732726116343.meta.1732726118359.meta, exclude list is [], retry=0 2024-11-27T16:48:38,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:38,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:38,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:38,383 INFO [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,40989,1732726116343/37bfedfd9766%2C40989%2C1732726116343.meta.1732726118359.meta 2024-11-27T16:48:38,384 DEBUG [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:42883:42883)] 2024-11-27T16:48:38,384 DEBUG [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:48:38,387 DEBUG [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T16:48:38,391 DEBUG [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T16:48:38,397 INFO [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T16:48:38,404 DEBUG [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T16:48:38,404 DEBUG [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:38,405 DEBUG [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T16:48:38,405 DEBUG [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T16:48:38,408 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T16:48:38,410 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T16:48:38,410 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:38,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T16:48:38,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T16:48:38,413 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T16:48:38,413 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:38,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T16:48:38,414 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T16:48:38,415 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T16:48:38,415 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:38,416 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T16:48:38,416 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T16:48:38,417 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T16:48:38,417 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:38,418 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T16:48:38,418 DEBUG [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T16:48:38,420 DEBUG [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740 2024-11-27T16:48:38,423 DEBUG [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740 2024-11-27T16:48:38,425 DEBUG [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T16:48:38,426 DEBUG [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T16:48:38,427 DEBUG [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T16:48:38,430 DEBUG [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T16:48:38,431 INFO [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75198927, jitterRate=0.12055133283138275}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T16:48:38,432 DEBUG [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T16:48:38,434 DEBUG [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732726118405Writing region info on filesystem at 1732726118406 (+1 ms)Initializing all the Stores at 1732726118408 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732726118408Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732726118408Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726118408Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732726118408Cleaning up temporary data from old regions at 1732726118426 (+18 ms)Running coprocessor post-open hooks at 1732726118432 (+6 ms)Region opened successfully at 1732726118434 (+2 ms) 2024-11-27T16:48:38,443 INFO [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732726118329 2024-11-27T16:48:38,457 DEBUG [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T16:48:38,457 INFO [RS_OPEN_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T16:48:38,459 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=37bfedfd9766,40989,1732726116343 2024-11-27T16:48:38,461 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 37bfedfd9766,40989,1732726116343, state=OPEN 2024-11-27T16:48:38,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37003-0x10039e202d30002, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T16:48:38,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T16:48:38,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T16:48:38,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T16:48:38,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T16:48:38,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T16:48:38,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T16:48:38,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T16:48:38,465 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=37bfedfd9766,40989,1732726116343 2024-11-27T16:48:38,471 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T16:48:38,471 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=37bfedfd9766,40989,1732726116343 in 305 msec 2024-11-27T16:48:38,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T16:48:38,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 782 msec 2024-11-27T16:48:38,480 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T16:48:38,480 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T16:48:38,532 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T16:48:38,534 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37bfedfd9766,40989,1732726116343, seqNum=-1] 2024-11-27T16:48:38,564 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T16:48:38,568 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42435, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T16:48:38,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1760 sec 2024-11-27T16:48:38,593 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732726118592, completionTime=-1 2024-11-27T16:48:38,596 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-27T16:48:38,596 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T16:48:38,629 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-27T16:48:38,630 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732726178630 2024-11-27T16:48:38,630 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732726238630 2024-11-27T16:48:38,630 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 33 msec 2024-11-27T16:48:38,632 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-27T16:48:38,641 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37bfedfd9766,35959,1732726115618-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:38,641 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37bfedfd9766,35959,1732726115618-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:38,641 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37bfedfd9766,35959,1732726115618-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:38,643 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-37bfedfd9766:35959, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:38,644 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:38,644 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:38,657 DEBUG [master/37bfedfd9766:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T16:48:38,683 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.103sec 2024-11-27T16:48:38,685 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T16:48:38,687 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T16:48:38,688 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T16:48:38,689 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T16:48:38,689 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T16:48:38,690 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37bfedfd9766,35959,1732726115618-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T16:48:38,690 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37bfedfd9766,35959,1732726115618-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T16:48:38,696 DEBUG [master/37bfedfd9766:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T16:48:38,697 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T16:48:38,698 INFO [master/37bfedfd9766:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37bfedfd9766,35959,1732726115618-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:38,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4810dfd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T16:48:38,744 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37bfedfd9766,35959,-1 for getting cluster id 2024-11-27T16:48:38,747 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T16:48:38,756 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7a7dfc1d-b607-4ef1-a912-89ad055552df' 2024-11-27T16:48:38,758 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T16:48:38,758 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7a7dfc1d-b607-4ef1-a912-89ad055552df" 2024-11-27T16:48:38,758 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5042c366, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T16:48:38,758 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37bfedfd9766,35959,-1] 2024-11-27T16:48:38,761 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T16:48:38,763 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T16:48:38,764 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T16:48:38,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@91db148, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T16:48:38,768 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T16:48:38,775 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37bfedfd9766,40989,1732726116343, seqNum=-1] 2024-11-27T16:48:38,776 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T16:48:38,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T16:48:38,807 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=37bfedfd9766,35959,1732726115618 2024-11-27T16:48:38,807 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:40409/hbase 2024-11-27T16:48:38,829 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=366, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=263, ProcessCount=11, AvailableMemoryMB=7007 2024-11-27T16:48:38,855 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:38,859 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:38,860 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T16:48:38,865 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-30929677, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/hregion-30929677, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:38,888 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-30929677/hregion-30929677.1732726118867, exclude list is [], retry=0 2024-11-27T16:48:38,893 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:38,893 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:38,894 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:38,897 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-30929677/hregion-30929677.1732726118867 2024-11-27T16:48:38,900 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:39655:39655)] 2024-11-27T16:48:38,900 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1e2f7c5dd6a4cac2d445314e1ad3a770, NAME => 'testReplayEditsWrittenIntoWAL,,1732726118856.1e2f7c5dd6a4cac2d445314e1ad3a770.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40409/hbase 2024-11-27T16:48:38,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741838_1014 (size=64) 2024-11-27T16:48:38,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741838_1014 (size=64) 2024-11-27T16:48:38,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741838_1014 (size=64) 2024-11-27T16:48:38,921 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732726118856.1e2f7c5dd6a4cac2d445314e1ad3a770.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:38,923 INFO [StoreOpener-1e2f7c5dd6a4cac2d445314e1ad3a770-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1e2f7c5dd6a4cac2d445314e1ad3a770 2024-11-27T16:48:38,926 INFO [StoreOpener-1e2f7c5dd6a4cac2d445314e1ad3a770-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e2f7c5dd6a4cac2d445314e1ad3a770 columnFamilyName a 2024-11-27T16:48:38,926 DEBUG [StoreOpener-1e2f7c5dd6a4cac2d445314e1ad3a770-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:38,926 INFO [StoreOpener-1e2f7c5dd6a4cac2d445314e1ad3a770-1 {}] regionserver.HStore(327): Store=1e2f7c5dd6a4cac2d445314e1ad3a770/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:38,927 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1e2f7c5dd6a4cac2d445314e1ad3a770 2024-11-27T16:48:38,928 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770 2024-11-27T16:48:38,929 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770 2024-11-27T16:48:38,930 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1e2f7c5dd6a4cac2d445314e1ad3a770 2024-11-27T16:48:38,930 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1e2f7c5dd6a4cac2d445314e1ad3a770 2024-11-27T16:48:38,933 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1e2f7c5dd6a4cac2d445314e1ad3a770 2024-11-27T16:48:38,937 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T16:48:38,938 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1e2f7c5dd6a4cac2d445314e1ad3a770; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59233856, jitterRate=-0.11734676361083984}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T16:48:38,939 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1e2f7c5dd6a4cac2d445314e1ad3a770: Writing region info on filesystem at 1732726118921Initializing all the Stores at 1732726118923 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726118923Cleaning up temporary data from old regions at 1732726118930 (+7 ms)Region opened successfully at 1732726118939 (+9 ms) 2024-11-27T16:48:38,939 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1e2f7c5dd6a4cac2d445314e1ad3a770, disabling compactions & flushes 2024-11-27T16:48:38,939 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732726118856.1e2f7c5dd6a4cac2d445314e1ad3a770. 2024-11-27T16:48:38,939 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732726118856.1e2f7c5dd6a4cac2d445314e1ad3a770. 2024-11-27T16:48:38,940 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732726118856.1e2f7c5dd6a4cac2d445314e1ad3a770. after waiting 0 ms 2024-11-27T16:48:38,940 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732726118856.1e2f7c5dd6a4cac2d445314e1ad3a770. 2024-11-27T16:48:38,940 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732726118856.1e2f7c5dd6a4cac2d445314e1ad3a770. 2024-11-27T16:48:38,940 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1e2f7c5dd6a4cac2d445314e1ad3a770: Waiting for close lock at 1732726118939Disabling compacts and flushes for region at 1732726118939Disabling writes for close at 1732726118940 (+1 ms)Writing region close event to WAL at 1732726118940Closed at 1732726118940 2024-11-27T16:48:38,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741837_1013 (size=95) 2024-11-27T16:48:38,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741837_1013 (size=95) 2024-11-27T16:48:38,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741837_1013 (size=95) 2024-11-27T16:48:38,953 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T16:48:38,953 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-30929677:(num 1732726118867) 2024-11-27T16:48:38,954 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-27T16:48:38,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741839_1015 (size=320) 2024-11-27T16:48:38,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741839_1015 (size=320) 2024-11-27T16:48:38,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741839_1015 (size=320) 2024-11-27T16:48:38,977 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-27T16:48:38,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741840_1016 (size=253) 2024-11-27T16:48:38,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741840_1016 (size=253) 2024-11-27T16:48:38,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741840_1016 (size=253) 2024-11-27T16:48:39,011 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-1, size=320 (320bytes) 2024-11-27T16:48:39,012 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-27T16:48:39,012 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-27T16:48:39,012 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-1 2024-11-27T16:48:39,018 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-1 after 4ms 2024-11-27T16:48:39,023 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:39,024 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-1 took 15ms 2024-11-27T16:48:39,035 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-1 so closing down 2024-11-27T16:48:39,035 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T16:48:39,038 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-27T16:48:39,041 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/0000000000000000001-wal-1.temp 2024-11-27T16:48:39,043 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T16:48:39,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741841_1017 (size=320) 2024-11-27T16:48:39,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741841_1017 (size=320) 2024-11-27T16:48:39,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741841_1017 (size=320) 2024-11-27T16:48:39,056 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-27T16:48:39,059 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/0000000000000000002 2024-11-27T16:48:39,064 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-27T16:48:39,064 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-1, journal: Splitting hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-1, size=320 (320bytes) at 1732726119011Finishing writing output for hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-1 so closing down at 1732726119035 (+24 ms)Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/0000000000000000001-wal-1.temp at 1732726119041 (+6 ms)3 split writer threads finished at 1732726119043 (+2 ms)Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732726119056 (+13 ms)Rename recovered edits hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/0000000000000000002 at 1732726119059 (+3 ms)Processed 2 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1732726119064 (+5 ms) 2024-11-27T16:48:39,088 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-2, size=253 (253bytes) 2024-11-27T16:48:39,089 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-2 2024-11-27T16:48:39,090 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-2 after 1ms 2024-11-27T16:48:39,094 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:39,095 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-2 took 7ms 2024-11-27T16:48:39,098 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-2 so closing down 2024-11-27T16:48:39,098 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T16:48:39,102 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-27T16:48:39,103 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/0000000000000000002-wal-2.temp 2024-11-27T16:48:39,104 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T16:48:39,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741842_1018 (size=253) 2024-11-27T16:48:39,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741842_1018 (size=253) 2024-11-27T16:48:39,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741842_1018 (size=253) 2024-11-27T16:48:39,115 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-27T16:48:39,120 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:39,123 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/0000000000000000002-wal-2.temp, length=253 2024-11-27T16:48:39,124 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 29 ms; skipped=0; WAL=hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-27T16:48:39,125 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-2, journal: Splitting hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-2, size=253 (253bytes) at 1732726119089Finishing writing output for hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-2 so closing down at 1732726119098 (+9 ms)Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/0000000000000000002-wal-2.temp at 1732726119103 (+5 ms)3 split writer threads finished at 1732726119104 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732726119115 (+11 ms)Processed 1 edits across 1 Regions in 29 ms; skipped=0; WAL=hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1732726119124 (+9 ms) 2024-11-27T16:48:39,125 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:48:39,127 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:39,142 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal.1732726119128, exclude list is [], retry=0 2024-11-27T16:48:39,146 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:39,147 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:39,148 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:39,150 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal.1732726119128 2024-11-27T16:48:39,152 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:33829:33829)] 2024-11-27T16:48:39,152 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e2f7c5dd6a4cac2d445314e1ad3a770, NAME => 'testReplayEditsWrittenIntoWAL,,1732726118856.1e2f7c5dd6a4cac2d445314e1ad3a770.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:48:39,152 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732726118856.1e2f7c5dd6a4cac2d445314e1ad3a770.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:39,152 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1e2f7c5dd6a4cac2d445314e1ad3a770 2024-11-27T16:48:39,152 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1e2f7c5dd6a4cac2d445314e1ad3a770 2024-11-27T16:48:39,154 INFO [StoreOpener-1e2f7c5dd6a4cac2d445314e1ad3a770-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1e2f7c5dd6a4cac2d445314e1ad3a770 2024-11-27T16:48:39,156 INFO [StoreOpener-1e2f7c5dd6a4cac2d445314e1ad3a770-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e2f7c5dd6a4cac2d445314e1ad3a770 columnFamilyName a 2024-11-27T16:48:39,156 DEBUG [StoreOpener-1e2f7c5dd6a4cac2d445314e1ad3a770-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:39,157 INFO [StoreOpener-1e2f7c5dd6a4cac2d445314e1ad3a770-1 {}] regionserver.HStore(327): Store=1e2f7c5dd6a4cac2d445314e1ad3a770/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:39,157 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1e2f7c5dd6a4cac2d445314e1ad3a770 2024-11-27T16:48:39,158 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770 2024-11-27T16:48:39,161 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770 2024-11-27T16:48:39,163 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/0000000000000000002 2024-11-27T16:48:39,166 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:39,175 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/0000000000000000002 2024-11-27T16:48:39,179 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1e2f7c5dd6a4cac2d445314e1ad3a770 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-27T16:48:39,233 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/.tmp/a/d9e98ed24ffe40f9b53184c29da0d60b is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732726118953/Put/seqid=0 2024-11-27T16:48:39,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741844_1020 (size=5170) 2024-11-27T16:48:39,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741844_1020 (size=5170) 2024-11-27T16:48:39,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741844_1020 (size=5170) 2024-11-27T16:48:39,250 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/.tmp/a/d9e98ed24ffe40f9b53184c29da0d60b 2024-11-27T16:48:39,297 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/.tmp/a/d9e98ed24ffe40f9b53184c29da0d60b as hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/a/d9e98ed24ffe40f9b53184c29da0d60b 2024-11-27T16:48:39,306 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/a/d9e98ed24ffe40f9b53184c29da0d60b, entries=2, sequenceid=2, filesize=5.0 K 2024-11-27T16:48:39,311 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1e2f7c5dd6a4cac2d445314e1ad3a770 in 132ms, sequenceid=2, compaction requested=false; wal=null 2024-11-27T16:48:39,313 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/0000000000000000002 2024-11-27T16:48:39,313 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1e2f7c5dd6a4cac2d445314e1ad3a770 2024-11-27T16:48:39,314 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1e2f7c5dd6a4cac2d445314e1ad3a770 2024-11-27T16:48:39,316 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1e2f7c5dd6a4cac2d445314e1ad3a770 2024-11-27T16:48:39,320 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/1e2f7c5dd6a4cac2d445314e1ad3a770/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-27T16:48:39,321 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1e2f7c5dd6a4cac2d445314e1ad3a770; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60353680, jitterRate=-0.10066008567810059}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T16:48:39,322 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1e2f7c5dd6a4cac2d445314e1ad3a770: Writing region info on filesystem at 1732726119152Initializing all the Stores at 1732726119154 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726119154Obtaining lock to block concurrent updates at 1732726119179 (+25 ms)Preparing flush snapshotting stores in 1e2f7c5dd6a4cac2d445314e1ad3a770 at 1732726119179Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732726118856.1e2f7c5dd6a4cac2d445314e1ad3a770., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732726119183 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732726118856.1e2f7c5dd6a4cac2d445314e1ad3a770. at 1732726119183Flushing 1e2f7c5dd6a4cac2d445314e1ad3a770/a: creating writer at 1732726119184 (+1 ms)Flushing 1e2f7c5dd6a4cac2d445314e1ad3a770/a: appending metadata at 1732726119223 (+39 ms)Flushing 1e2f7c5dd6a4cac2d445314e1ad3a770/a: closing flushed file at 1732726119225 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b72f05d: reopening flushed file at 1732726119295 (+70 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1e2f7c5dd6a4cac2d445314e1ad3a770 in 132ms, sequenceid=2, compaction requested=false; wal=null at 1732726119311 (+16 ms)Cleaning up temporary data from old regions at 1732726119314 (+3 ms)Region opened successfully at 1732726119322 (+8 ms) 2024-11-27T16:48:39,349 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=377 (was 366) Potentially hanging thread: PacketResponder: BP-89784467-172.17.0.2-1732726112257:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:40409/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:37724 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-89784467-172.17.0.2-1732726112257:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:35322 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-89784467-172.17.0.2-1732726112257:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:35418 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:55130 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:37880 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:54950 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=263 (was 263), ProcessCount=11 (was 11), AvailableMemoryMB=6990 (was 7007) 2024-11-27T16:48:39,360 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=377, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=263, ProcessCount=11, AvailableMemoryMB=6990 2024-11-27T16:48:39,376 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:39,378 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:39,379 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T16:48:39,384 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-19915197, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/hregion-19915197, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:39,406 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-19915197/hregion-19915197.1732726119385, exclude list is [], retry=0 2024-11-27T16:48:39,410 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:39,410 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:39,411 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:39,415 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-19915197/hregion-19915197.1732726119385 2024-11-27T16:48:39,416 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:39655:39655)] 2024-11-27T16:48:39,416 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3defc3509955832b3563083818e5192e, NAME => 'testReplayEditsWrittenIntoWAL,,1732726119377.3defc3509955832b3563083818e5192e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40409/hbase 2024-11-27T16:48:39,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741846_1022 (size=64) 2024-11-27T16:48:39,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741846_1022 (size=64) 2024-11-27T16:48:39,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741846_1022 (size=64) 2024-11-27T16:48:39,430 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732726119377.3defc3509955832b3563083818e5192e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:39,432 INFO [StoreOpener-3defc3509955832b3563083818e5192e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3defc3509955832b3563083818e5192e 2024-11-27T16:48:39,434 INFO [StoreOpener-3defc3509955832b3563083818e5192e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3defc3509955832b3563083818e5192e columnFamilyName a 2024-11-27T16:48:39,434 DEBUG [StoreOpener-3defc3509955832b3563083818e5192e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:39,435 INFO [StoreOpener-3defc3509955832b3563083818e5192e-1 {}] regionserver.HStore(327): Store=3defc3509955832b3563083818e5192e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:39,435 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3defc3509955832b3563083818e5192e 2024-11-27T16:48:39,437 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e 2024-11-27T16:48:39,437 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e 2024-11-27T16:48:39,438 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3defc3509955832b3563083818e5192e 2024-11-27T16:48:39,438 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3defc3509955832b3563083818e5192e 2024-11-27T16:48:39,440 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3defc3509955832b3563083818e5192e 2024-11-27T16:48:39,444 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T16:48:39,445 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3defc3509955832b3563083818e5192e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72491947, jitterRate=0.08021418750286102}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T16:48:39,445 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3defc3509955832b3563083818e5192e: Writing region info on filesystem at 1732726119430Initializing all the Stores at 1732726119431 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726119431Cleaning up temporary data from old regions at 1732726119438 (+7 ms)Region opened successfully at 1732726119445 (+7 ms) 2024-11-27T16:48:39,445 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3defc3509955832b3563083818e5192e, disabling compactions & flushes 2024-11-27T16:48:39,445 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732726119377.3defc3509955832b3563083818e5192e. 2024-11-27T16:48:39,445 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732726119377.3defc3509955832b3563083818e5192e. 2024-11-27T16:48:39,445 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732726119377.3defc3509955832b3563083818e5192e. after waiting 0 ms 2024-11-27T16:48:39,445 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732726119377.3defc3509955832b3563083818e5192e. 2024-11-27T16:48:39,446 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732726119377.3defc3509955832b3563083818e5192e. 2024-11-27T16:48:39,446 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3defc3509955832b3563083818e5192e: Waiting for close lock at 1732726119445Disabling compacts and flushes for region at 1732726119445Disabling writes for close at 1732726119445Writing region close event to WAL at 1732726119446 (+1 ms)Closed at 1732726119446 2024-11-27T16:48:39,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741845_1021 (size=95) 2024-11-27T16:48:39,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741845_1021 (size=95) 2024-11-27T16:48:39,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741845_1021 (size=95) 2024-11-27T16:48:39,454 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T16:48:39,454 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-19915197:(num 1732726119385) 2024-11-27T16:48:39,456 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-27T16:48:39,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741847_1023 (size=320) 2024-11-27T16:48:39,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741847_1023 (size=320) 2024-11-27T16:48:39,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741847_1023 (size=320) 2024-11-27T16:48:39,472 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-27T16:48:39,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741848_1024 (size=253) 2024-11-27T16:48:39,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741848_1024 (size=253) 2024-11-27T16:48:39,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741848_1024 (size=253) 2024-11-27T16:48:39,502 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-2, size=253 (253bytes) 2024-11-27T16:48:39,502 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-2 2024-11-27T16:48:39,503 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-2 after 1ms 2024-11-27T16:48:39,506 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:39,507 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-2 took 5ms 2024-11-27T16:48:39,509 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-2 so closing down 2024-11-27T16:48:39,509 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T16:48:39,511 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-27T16:48:39,512 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000002-wal-2.temp 2024-11-27T16:48:39,513 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T16:48:39,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741849_1025 (size=253) 2024-11-27T16:48:39,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741849_1025 (size=253) 2024-11-27T16:48:39,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741849_1025 (size=253) 2024-11-27T16:48:39,521 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-27T16:48:39,523 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000002 2024-11-27T16:48:39,523 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-27T16:48:39,524 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-2, journal: Splitting hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-2, size=253 (253bytes) at 1732726119502Finishing writing output for hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-2 so closing down at 1732726119509 (+7 ms)Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000002-wal-2.temp at 1732726119512 (+3 ms)3 split writer threads finished at 1732726119513 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732726119521 (+8 ms)Rename recovered edits hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000002 at 1732726119523 (+2 ms)Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1732726119523 2024-11-27T16:48:39,539 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-1, size=320 (320bytes) 2024-11-27T16:48:39,539 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-1 2024-11-27T16:48:39,539 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-1 after 0ms 2024-11-27T16:48:39,543 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:39,543 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-1 took 5ms 2024-11-27T16:48:39,545 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-1 so closing down 2024-11-27T16:48:39,546 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T16:48:39,547 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-27T16:48:39,549 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000001-wal-1.temp 2024-11-27T16:48:39,549 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T16:48:39,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741850_1026 (size=320) 2024-11-27T16:48:39,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741850_1026 (size=320) 2024-11-27T16:48:39,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741850_1026 (size=320) 2024-11-27T16:48:39,558 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-27T16:48:39,562 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:39,564 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000002, length=253 2024-11-27T16:48:39,566 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000002 2024-11-27T16:48:39,567 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-27T16:48:39,567 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-1, journal: Splitting hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-1, size=320 (320bytes) at 1732726119539Finishing writing output for hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-1 so closing down at 1732726119545 (+6 ms)Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000001-wal-1.temp at 1732726119549 (+4 ms)3 split writer threads finished at 1732726119549Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732726119558 (+9 ms)Rename recovered edits hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000002 at 1732726119567 (+9 ms)Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1732726119567 2024-11-27T16:48:39,567 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:48:39,570 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:39,591 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal.1732726119571, exclude list is [], retry=0 2024-11-27T16:48:39,595 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:39,596 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:39,596 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:39,600 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal.1732726119571 2024-11-27T16:48:39,601 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:33829:33829)] 2024-11-27T16:48:39,601 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3defc3509955832b3563083818e5192e, NAME => 'testReplayEditsWrittenIntoWAL,,1732726119377.3defc3509955832b3563083818e5192e.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:48:39,601 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732726119377.3defc3509955832b3563083818e5192e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:39,601 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3defc3509955832b3563083818e5192e 2024-11-27T16:48:39,601 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3defc3509955832b3563083818e5192e 2024-11-27T16:48:39,603 INFO [StoreOpener-3defc3509955832b3563083818e5192e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3defc3509955832b3563083818e5192e 2024-11-27T16:48:39,606 INFO [StoreOpener-3defc3509955832b3563083818e5192e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3defc3509955832b3563083818e5192e columnFamilyName a 2024-11-27T16:48:39,606 DEBUG [StoreOpener-3defc3509955832b3563083818e5192e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:39,607 INFO [StoreOpener-3defc3509955832b3563083818e5192e-1 {}] regionserver.HStore(327): Store=3defc3509955832b3563083818e5192e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:39,607 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3defc3509955832b3563083818e5192e 2024-11-27T16:48:39,608 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e 2024-11-27T16:48:39,610 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e 2024-11-27T16:48:39,611 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000002 2024-11-27T16:48:39,613 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:39,615 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000002 2024-11-27T16:48:39,615 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3defc3509955832b3563083818e5192e 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-27T16:48:39,632 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/.tmp/a/a60d6d6bcd264cdf8b5295b897b04dcf is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732726119454/Put/seqid=0 2024-11-27T16:48:39,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741852_1028 (size=5170) 2024-11-27T16:48:39,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741852_1028 (size=5170) 2024-11-27T16:48:39,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741852_1028 (size=5170) 2024-11-27T16:48:39,643 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/.tmp/a/a60d6d6bcd264cdf8b5295b897b04dcf 2024-11-27T16:48:39,652 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/.tmp/a/a60d6d6bcd264cdf8b5295b897b04dcf as hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/a/a60d6d6bcd264cdf8b5295b897b04dcf 2024-11-27T16:48:39,660 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/a/a60d6d6bcd264cdf8b5295b897b04dcf, entries=2, sequenceid=2, filesize=5.0 K 2024-11-27T16:48:39,661 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 3defc3509955832b3563083818e5192e in 45ms, sequenceid=2, compaction requested=false; wal=null 2024-11-27T16:48:39,661 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/0000000000000000002 2024-11-27T16:48:39,662 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3defc3509955832b3563083818e5192e 2024-11-27T16:48:39,662 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3defc3509955832b3563083818e5192e 2024-11-27T16:48:39,665 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3defc3509955832b3563083818e5192e 2024-11-27T16:48:39,667 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/3defc3509955832b3563083818e5192e/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-27T16:48:39,668 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3defc3509955832b3563083818e5192e; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63278887, jitterRate=-0.057071104645729065}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T16:48:39,669 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3defc3509955832b3563083818e5192e: Writing region info on filesystem at 1732726119601Initializing all the Stores at 1732726119603 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726119603Obtaining lock to block concurrent updates at 1732726119615 (+12 ms)Preparing flush snapshotting stores in 3defc3509955832b3563083818e5192e at 1732726119615Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732726119377.3defc3509955832b3563083818e5192e., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732726119615Flushing stores of testReplayEditsWrittenIntoWAL,,1732726119377.3defc3509955832b3563083818e5192e. at 1732726119615Flushing 3defc3509955832b3563083818e5192e/a: creating writer at 1732726119615Flushing 3defc3509955832b3563083818e5192e/a: appending metadata at 1732726119631 (+16 ms)Flushing 3defc3509955832b3563083818e5192e/a: closing flushed file at 1732726119631Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7eab785a: reopening flushed file at 1732726119651 (+20 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 3defc3509955832b3563083818e5192e in 45ms, sequenceid=2, compaction requested=false; wal=null at 1732726119661 (+10 ms)Cleaning up temporary data from old regions at 1732726119662 (+1 ms)Region opened successfully at 1732726119668 (+6 ms) 2024-11-27T16:48:39,686 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=387 (was 377) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:37954 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-89784467-172.17.0.2-1732726112257:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:35322 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:35510 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-89784467-172.17.0.2-1732726112257:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:54950 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:55206 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-89784467-172.17.0.2-1732726112257:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=263 (was 263), ProcessCount=11 (was 11), AvailableMemoryMB=6986 (was 6990) 2024-11-27T16:48:39,699 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=387, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=263, ProcessCount=11, AvailableMemoryMB=6985 2024-11-27T16:48:39,715 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:39,717 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:39,718 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T16:48:39,721 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-89045920, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/hregion-89045920, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:39,736 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-89045920/hregion-89045920.1732726119722, exclude list is [], retry=0 2024-11-27T16:48:39,740 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:39,740 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:39,741 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:39,744 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-89045920/hregion-89045920.1732726119722 2024-11-27T16:48:39,744 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:42883:42883)] 2024-11-27T16:48:39,745 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 183e8876db74679aa65f88c47e844bc5, NAME => 'testReplayEditsWrittenIntoWAL,,1732726119715.183e8876db74679aa65f88c47e844bc5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40409/hbase 2024-11-27T16:48:39,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741854_1030 (size=64) 2024-11-27T16:48:39,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741854_1030 (size=64) 2024-11-27T16:48:39,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741854_1030 (size=64) 2024-11-27T16:48:39,759 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732726119715.183e8876db74679aa65f88c47e844bc5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:39,761 INFO [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 183e8876db74679aa65f88c47e844bc5 2024-11-27T16:48:39,763 INFO [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 183e8876db74679aa65f88c47e844bc5 columnFamilyName a 2024-11-27T16:48:39,763 DEBUG [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:39,764 INFO [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] regionserver.HStore(327): Store=183e8876db74679aa65f88c47e844bc5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:39,764 INFO [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 183e8876db74679aa65f88c47e844bc5 2024-11-27T16:48:39,765 INFO [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 183e8876db74679aa65f88c47e844bc5 columnFamilyName b 2024-11-27T16:48:39,765 DEBUG [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:39,766 INFO [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] regionserver.HStore(327): Store=183e8876db74679aa65f88c47e844bc5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:39,766 INFO [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 183e8876db74679aa65f88c47e844bc5 2024-11-27T16:48:39,768 INFO [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 183e8876db74679aa65f88c47e844bc5 columnFamilyName c 2024-11-27T16:48:39,768 DEBUG [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:39,769 INFO [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] regionserver.HStore(327): Store=183e8876db74679aa65f88c47e844bc5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:39,769 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 183e8876db74679aa65f88c47e844bc5 2024-11-27T16:48:39,770 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5 2024-11-27T16:48:39,770 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5 2024-11-27T16:48:39,772 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 183e8876db74679aa65f88c47e844bc5 2024-11-27T16:48:39,772 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 183e8876db74679aa65f88c47e844bc5 2024-11-27T16:48:39,772 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T16:48:39,774 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 183e8876db74679aa65f88c47e844bc5 2024-11-27T16:48:39,776 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T16:48:39,777 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 183e8876db74679aa65f88c47e844bc5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73516489, jitterRate=0.09548105299472809}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T16:48:39,777 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 183e8876db74679aa65f88c47e844bc5: Writing region info on filesystem at 1732726119760Initializing all the Stores at 1732726119761 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726119761Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726119761Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726119761Cleaning up temporary data from old regions at 1732726119772 (+11 ms)Region opened successfully at 1732726119777 (+5 ms) 2024-11-27T16:48:39,777 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 183e8876db74679aa65f88c47e844bc5, disabling compactions & flushes 2024-11-27T16:48:39,777 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732726119715.183e8876db74679aa65f88c47e844bc5. 2024-11-27T16:48:39,777 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732726119715.183e8876db74679aa65f88c47e844bc5. 2024-11-27T16:48:39,777 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732726119715.183e8876db74679aa65f88c47e844bc5. after waiting 0 ms 2024-11-27T16:48:39,777 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732726119715.183e8876db74679aa65f88c47e844bc5. 2024-11-27T16:48:39,778 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732726119715.183e8876db74679aa65f88c47e844bc5. 2024-11-27T16:48:39,778 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 183e8876db74679aa65f88c47e844bc5: Waiting for close lock at 1732726119777Disabling compacts and flushes for region at 1732726119777Disabling writes for close at 1732726119777Writing region close event to WAL at 1732726119778 (+1 ms)Closed at 1732726119778 2024-11-27T16:48:39,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741853_1029 (size=95) 2024-11-27T16:48:39,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741853_1029 (size=95) 2024-11-27T16:48:39,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741853_1029 (size=95) 2024-11-27T16:48:39,784 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T16:48:39,784 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-89045920:(num 1732726119722) 2024-11-27T16:48:39,785 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:48:39,787 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:39,802 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726119787, exclude list is [], retry=0 2024-11-27T16:48:39,805 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:39,805 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:39,806 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:39,808 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726119787 2024-11-27T16:48:39,809 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:33829:33829)] 2024-11-27T16:48:40,062 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726119787, size=0 (0bytes) 2024-11-27T16:48:40,063 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726119787 might be still open, length is 0 2024-11-27T16:48:40,063 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726119787 2024-11-27T16:48:40,064 WARN [IPC Server handler 2 on default port 40409 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726119787 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-27T16:48:40,064 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726119787 after 1ms 2024-11-27T16:48:40,472 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:55236 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37433:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55236 dst: /127.0.0.1:37433 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37433 remote=/127.0.0.1:55236]. Total timeout mills is 60000, 59557 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:48:40,473 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:35548 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37511:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35548 dst: /127.0.0.1:37511 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:48:40,473 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:37996 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41739:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37996 dst: /127.0.0.1:41739 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:48:40,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741855_1032 (size=263633) 2024-11-27T16:48:40,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741855_1032 (size=263633) 2024-11-27T16:48:40,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741855_1032 (size=263633) 2024-11-27T16:48:43,913 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T16:48:43,965 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T16:48:44,065 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726119787 after 4002ms 2024-11-27T16:48:44,070 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726119787: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:44,072 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726119787 took 4010ms 2024-11-27T16:48:44,077 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732726119787.temp 2024-11-27T16:48:44,084 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/recovered.edits/0000000000000000001-wal.1732726119787.temp 2024-11-27T16:48:44,204 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726119787; continuing. 2024-11-27T16:48:44,204 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726119787 so closing down 2024-11-27T16:48:44,204 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T16:48:44,205 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T16:48:44,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741856_1033 (size=263641) 2024-11-27T16:48:44,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741856_1033 (size=263641) 2024-11-27T16:48:44,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741856_1033 (size=263641) 2024-11-27T16:48:44,216 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/recovered.edits/0000000000000000001-wal.1732726119787.temp (wrote 3002 edits, skipped 0 edits in 70 ms) 2024-11-27T16:48:44,218 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/recovered.edits/0000000000000000001-wal.1732726119787.temp to hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/recovered.edits/0000000000000003002 2024-11-27T16:48:44,218 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 145 ms; skipped=0; WAL=hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726119787, size=0, length=0, corrupted=false, cancelled=false 2024-11-27T16:48:44,218 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726119787, journal: Splitting hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726119787, size=0 (0bytes) at 1732726120063Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/recovered.edits/0000000000000000001-wal.1732726119787.temp at 1732726124084 (+4021 ms)Split 1024 edits, skipped 0 edits. at 1732726124153 (+69 ms)Split 2048 edits, skipped 0 edits. at 1732726124179 (+26 ms)Finishing writing output for hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726119787 so closing down at 1732726124204 (+25 ms)3 split writer threads finished at 1732726124205 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/recovered.edits/0000000000000000001-wal.1732726119787.temp (wrote 3002 edits, skipped 0 edits in 70 ms) at 1732726124216 (+11 ms)Rename recovered edits hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/recovered.edits/0000000000000000001-wal.1732726119787.temp to hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/recovered.edits/0000000000000003002 at 1732726124218 (+2 ms)Processed 3002 edits across 1 Regions in 145 ms; skipped=0; WAL=hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726119787, size=0, length=0, corrupted=false, cancelled=false at 1732726124218 2024-11-27T16:48:44,220 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726119787 to hdfs://localhost:40409/hbase/oldWALs/wal.1732726119787 2024-11-27T16:48:44,222 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/recovered.edits/0000000000000003002 2024-11-27T16:48:44,222 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:48:44,224 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:44,240 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726124225, exclude list is [], retry=0 2024-11-27T16:48:44,243 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:44,244 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:44,244 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:44,247 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726124225 2024-11-27T16:48:44,247 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:39655:39655)] 2024-11-27T16:48:44,248 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732726119715.183e8876db74679aa65f88c47e844bc5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:44,249 INFO [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 183e8876db74679aa65f88c47e844bc5 2024-11-27T16:48:44,251 INFO [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 183e8876db74679aa65f88c47e844bc5 columnFamilyName a 2024-11-27T16:48:44,251 DEBUG [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:44,251 INFO [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] regionserver.HStore(327): Store=183e8876db74679aa65f88c47e844bc5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:44,252 INFO [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 183e8876db74679aa65f88c47e844bc5 2024-11-27T16:48:44,253 INFO [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 183e8876db74679aa65f88c47e844bc5 columnFamilyName b 2024-11-27T16:48:44,253 DEBUG [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:44,253 INFO [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] regionserver.HStore(327): Store=183e8876db74679aa65f88c47e844bc5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:44,253 INFO [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 183e8876db74679aa65f88c47e844bc5 2024-11-27T16:48:44,254 INFO [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 183e8876db74679aa65f88c47e844bc5 columnFamilyName c 2024-11-27T16:48:44,255 DEBUG [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:44,255 INFO [StoreOpener-183e8876db74679aa65f88c47e844bc5-1 {}] regionserver.HStore(327): Store=183e8876db74679aa65f88c47e844bc5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:44,255 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 183e8876db74679aa65f88c47e844bc5 2024-11-27T16:48:44,256 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5 2024-11-27T16:48:44,258 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5 2024-11-27T16:48:44,259 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/recovered.edits/0000000000000003002 2024-11-27T16:48:44,262 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:44,302 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-27T16:48:44,737 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 183e8876db74679aa65f88c47e844bc5 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-27T16:48:44,775 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/.tmp/a/4e260070337a426f8ad902bf1fb38fc6 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732726119816/Put/seqid=0 2024-11-27T16:48:44,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741858_1035 (size=50463) 2024-11-27T16:48:44,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741858_1035 (size=50463) 2024-11-27T16:48:44,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741858_1035 (size=50463) 2024-11-27T16:48:44,787 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/.tmp/a/4e260070337a426f8ad902bf1fb38fc6 2024-11-27T16:48:44,796 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/.tmp/a/4e260070337a426f8ad902bf1fb38fc6 as hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/a/4e260070337a426f8ad902bf1fb38fc6 2024-11-27T16:48:44,804 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/a/4e260070337a426f8ad902bf1fb38fc6, entries=754, sequenceid=754, filesize=49.3 K 2024-11-27T16:48:44,805 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 183e8876db74679aa65f88c47e844bc5 in 68ms, sequenceid=754, compaction requested=false; wal=null 2024-11-27T16:48:44,828 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-27T16:48:44,829 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 183e8876db74679aa65f88c47e844bc5 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-27T16:48:44,839 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/.tmp/a/d74bc42283b0488394067e789c17c486 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732726119871/Put/seqid=0 2024-11-27T16:48:44,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741859_1036 (size=20072) 2024-11-27T16:48:44,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741859_1036 (size=20072) 2024-11-27T16:48:44,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741859_1036 (size=20072) 2024-11-27T16:48:44,851 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/.tmp/a/d74bc42283b0488394067e789c17c486 2024-11-27T16:48:44,887 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/.tmp/b/d49e8110373f4feabb5ef91d1887792e is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732726119899/Put/seqid=0 2024-11-27T16:48:44,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741860_1037 (size=35835) 2024-11-27T16:48:44,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741860_1037 (size=35835) 2024-11-27T16:48:44,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741860_1037 (size=35835) 2024-11-27T16:48:44,898 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/.tmp/b/d49e8110373f4feabb5ef91d1887792e 2024-11-27T16:48:44,907 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/.tmp/a/d74bc42283b0488394067e789c17c486 as hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/a/d74bc42283b0488394067e789c17c486 2024-11-27T16:48:44,915 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/a/d74bc42283b0488394067e789c17c486, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-27T16:48:44,916 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/.tmp/b/d49e8110373f4feabb5ef91d1887792e as hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/b/d49e8110373f4feabb5ef91d1887792e 2024-11-27T16:48:44,925 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/b/d49e8110373f4feabb5ef91d1887792e, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-27T16:48:44,925 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 183e8876db74679aa65f88c47e844bc5 in 97ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-27T16:48:44,944 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-27T16:48:44,945 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 183e8876db74679aa65f88c47e844bc5 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-27T16:48:44,952 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/.tmp/b/0e4e5026d8094e6cb9fa1d826efe4152 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732726119926/Put/seqid=0 2024-11-27T16:48:44,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741861_1038 (size=35082) 2024-11-27T16:48:44,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741861_1038 (size=35082) 2024-11-27T16:48:44,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741861_1038 (size=35082) 2024-11-27T16:48:44,963 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/.tmp/b/0e4e5026d8094e6cb9fa1d826efe4152 2024-11-27T16:48:44,996 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/.tmp/c/38528e59befb40c6b8d630d45cc524ce is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732726119967/Put/seqid=0 2024-11-27T16:48:45,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741862_1039 (size=20825) 2024-11-27T16:48:45,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741862_1039 (size=20825) 2024-11-27T16:48:45,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741862_1039 (size=20825) 2024-11-27T16:48:45,013 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/.tmp/c/38528e59befb40c6b8d630d45cc524ce 2024-11-27T16:48:45,020 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/.tmp/b/0e4e5026d8094e6cb9fa1d826efe4152 as hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/b/0e4e5026d8094e6cb9fa1d826efe4152 2024-11-27T16:48:45,028 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/b/0e4e5026d8094e6cb9fa1d826efe4152, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-27T16:48:45,030 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/.tmp/c/38528e59befb40c6b8d630d45cc524ce as hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/c/38528e59befb40c6b8d630d45cc524ce 2024-11-27T16:48:45,037 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/c/38528e59befb40c6b8d630d45cc524ce, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-27T16:48:45,038 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 183e8876db74679aa65f88c47e844bc5 in 92ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-27T16:48:45,055 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732726120027/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732726119715.183e8876db74679aa65f88c47e844bc5. 2024-11-27T16:48:45,058 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/recovered.edits/0000000000000003002 2024-11-27T16:48:45,059 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-27T16:48:45,059 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 183e8876db74679aa65f88c47e844bc5 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-27T16:48:45,068 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/.tmp/c/3dcc047f53ce4dd5a66eebde5b5889ce is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732726119978/Put/seqid=0 2024-11-27T16:48:45,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741863_1040 (size=50301) 2024-11-27T16:48:45,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741863_1040 (size=50301) 2024-11-27T16:48:45,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741863_1040 (size=50301) 2024-11-27T16:48:45,081 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/.tmp/c/3dcc047f53ce4dd5a66eebde5b5889ce 2024-11-27T16:48:45,088 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3dcc047f53ce4dd5a66eebde5b5889ce 2024-11-27T16:48:45,090 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/.tmp/c/3dcc047f53ce4dd5a66eebde5b5889ce as hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/c/3dcc047f53ce4dd5a66eebde5b5889ce 2024-11-27T16:48:45,096 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3dcc047f53ce4dd5a66eebde5b5889ce 2024-11-27T16:48:45,096 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/c/3dcc047f53ce4dd5a66eebde5b5889ce, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-27T16:48:45,097 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 183e8876db74679aa65f88c47e844bc5 in 38ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-27T16:48:45,098 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/recovered.edits/0000000000000003002 2024-11-27T16:48:45,099 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 183e8876db74679aa65f88c47e844bc5 2024-11-27T16:48:45,099 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 183e8876db74679aa65f88c47e844bc5 2024-11-27T16:48:45,100 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T16:48:45,103 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 183e8876db74679aa65f88c47e844bc5 2024-11-27T16:48:45,106 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenIntoWAL/183e8876db74679aa65f88c47e844bc5/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-27T16:48:45,107 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 183e8876db74679aa65f88c47e844bc5; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66541779, jitterRate=-0.008450224995613098}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T16:48:45,107 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 183e8876db74679aa65f88c47e844bc5: Writing region info on filesystem at 1732726124248Initializing all the Stores at 1732726124249 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726124249Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726124249Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726124249Cleaning up temporary data from old regions at 1732726125099 (+850 ms)Region opened successfully at 1732726125107 (+8 ms) 2024-11-27T16:48:45,186 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 183e8876db74679aa65f88c47e844bc5, disabling compactions & flushes 2024-11-27T16:48:45,186 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732726119715.183e8876db74679aa65f88c47e844bc5. 2024-11-27T16:48:45,186 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732726119715.183e8876db74679aa65f88c47e844bc5. 2024-11-27T16:48:45,186 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732726119715.183e8876db74679aa65f88c47e844bc5. after waiting 0 ms 2024-11-27T16:48:45,186 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732726119715.183e8876db74679aa65f88c47e844bc5. 2024-11-27T16:48:45,189 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732726119715.183e8876db74679aa65f88c47e844bc5. 2024-11-27T16:48:45,189 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 183e8876db74679aa65f88c47e844bc5: Waiting for close lock at 1732726125186Disabling compacts and flushes for region at 1732726125186Disabling writes for close at 1732726125186Writing region close event to WAL at 1732726125189 (+3 ms)Closed at 1732726125189 2024-11-27T16:48:45,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741857_1034 (size=95) 2024-11-27T16:48:45,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741857_1034 (size=95) 2024-11-27T16:48:45,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741857_1034 (size=95) 2024-11-27T16:48:45,198 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T16:48:45,198 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732726124225) 2024-11-27T16:48:45,216 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=403 (was 387) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1085751279_22 at /127.0.0.1:35156 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1085751279_22 at /127.0.0.1:50304 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1085751279_22 at /127.0.0.1:46254 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:40409 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38561 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Client (1112330208) connection to localhost/127.0.0.1:40409 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1112330208) connection to localhost/127.0.0.1:43979 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43979 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1112330208) connection to localhost/127.0.0.1:38561 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1085751279_22 at /127.0.0.1:46218 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=250 (was 263), ProcessCount=11 (was 11), AvailableMemoryMB=6844 (was 6985) 2024-11-27T16:48:45,230 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=403, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=250, ProcessCount=11, AvailableMemoryMB=6843 2024-11-27T16:48:45,247 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:45,250 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:45,250 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T16:48:45,254 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-66287426, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/hregion-66287426, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:45,268 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-66287426/hregion-66287426.1732726125254, exclude list is [], retry=0 2024-11-27T16:48:45,271 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:45,272 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:45,272 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:45,274 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-66287426/hregion-66287426.1732726125254 2024-11-27T16:48:45,275 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:42883:42883)] 2024-11-27T16:48:45,275 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4b22ea80fe1fb41359a4dc815a485075, NAME => 'test2727,,1732726125248.4b22ea80fe1fb41359a4dc815a485075.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40409/hbase 2024-11-27T16:48:45,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741865_1042 (size=43) 2024-11-27T16:48:45,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741865_1042 (size=43) 2024-11-27T16:48:45,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741865_1042 (size=43) 2024-11-27T16:48:45,291 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732726125248.4b22ea80fe1fb41359a4dc815a485075.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:45,293 INFO [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:45,294 INFO [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b22ea80fe1fb41359a4dc815a485075 columnFamilyName a 2024-11-27T16:48:45,294 DEBUG [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:45,295 INFO [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] regionserver.HStore(327): Store=4b22ea80fe1fb41359a4dc815a485075/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:45,295 INFO [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:45,297 INFO [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b22ea80fe1fb41359a4dc815a485075 columnFamilyName b 2024-11-27T16:48:45,297 DEBUG [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:45,297 INFO [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] regionserver.HStore(327): Store=4b22ea80fe1fb41359a4dc815a485075/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:45,297 INFO [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:45,299 INFO [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b22ea80fe1fb41359a4dc815a485075 columnFamilyName c 2024-11-27T16:48:45,299 DEBUG [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:45,299 INFO [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] regionserver.HStore(327): Store=4b22ea80fe1fb41359a4dc815a485075/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:45,300 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:45,300 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:45,301 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:45,302 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:45,302 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:45,303 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T16:48:45,304 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:45,307 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T16:48:45,307 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4b22ea80fe1fb41359a4dc815a485075; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64978775, jitterRate=-0.03174079954624176}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T16:48:45,309 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4b22ea80fe1fb41359a4dc815a485075: Writing region info on filesystem at 1732726125291Initializing all the Stores at 1732726125292 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726125292Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726125293 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726125293Cleaning up temporary data from old regions at 1732726125302 (+9 ms)Region opened successfully at 1732726125309 (+7 ms) 2024-11-27T16:48:45,309 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4b22ea80fe1fb41359a4dc815a485075, disabling compactions & flushes 2024-11-27T16:48:45,309 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732726125248.4b22ea80fe1fb41359a4dc815a485075. 2024-11-27T16:48:45,309 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732726125248.4b22ea80fe1fb41359a4dc815a485075. 2024-11-27T16:48:45,309 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732726125248.4b22ea80fe1fb41359a4dc815a485075. after waiting 0 ms 2024-11-27T16:48:45,309 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732726125248.4b22ea80fe1fb41359a4dc815a485075. 2024-11-27T16:48:45,310 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732726125248.4b22ea80fe1fb41359a4dc815a485075. 2024-11-27T16:48:45,310 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4b22ea80fe1fb41359a4dc815a485075: Waiting for close lock at 1732726125309Disabling compacts and flushes for region at 1732726125309Disabling writes for close at 1732726125309Writing region close event to WAL at 1732726125310 (+1 ms)Closed at 1732726125310 2024-11-27T16:48:45,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741864_1041 (size=95) 2024-11-27T16:48:45,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741864_1041 (size=95) 2024-11-27T16:48:45,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741864_1041 (size=95) 2024-11-27T16:48:45,318 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T16:48:45,318 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-66287426:(num 1732726125254) 2024-11-27T16:48:45,318 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:48:45,320 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:45,335 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125320, exclude list is [], retry=0 2024-11-27T16:48:45,338 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:45,338 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:45,339 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:45,341 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125320 2024-11-27T16:48:45,342 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:39655:39655)] 2024-11-27T16:48:45,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741866_1043 (size=263359) 2024-11-27T16:48:45,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741866_1043 (size=263359) 2024-11-27T16:48:45,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741866_1043 (size=263359) 2024-11-27T16:48:45,570 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125320, size=257.2 K (263359bytes) 2024-11-27T16:48:45,571 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125320 2024-11-27T16:48:45,571 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125320 after 0ms 2024-11-27T16:48:45,575 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125320: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:45,577 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125320 took 7ms 2024-11-27T16:48:45,582 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732726125320.temp 2024-11-27T16:48:45,584 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000000001-wal.1732726125320.temp 2024-11-27T16:48:45,643 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125320 so closing down 2024-11-27T16:48:45,643 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T16:48:45,643 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T16:48:45,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741867_1044 (size=263359) 2024-11-27T16:48:45,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741867_1044 (size=263359) 2024-11-27T16:48:45,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741867_1044 (size=263359) 2024-11-27T16:48:45,653 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000000001-wal.1732726125320.temp (wrote 3000 edits, skipped 0 edits in 53 ms) 2024-11-27T16:48:45,655 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000000001-wal.1732726125320.temp to hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000003000 2024-11-27T16:48:45,655 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 78 ms; skipped=0; WAL=hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125320, size=257.2 K, length=263359, corrupted=false, cancelled=false 2024-11-27T16:48:45,655 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125320, journal: Splitting hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125320, size=257.2 K (263359bytes) at 1732726125570Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000000001-wal.1732726125320.temp at 1732726125584 (+14 ms)Split 1024 edits, skipped 0 edits. at 1732726125599 (+15 ms)Split 2048 edits, skipped 0 edits. at 1732726125620 (+21 ms)Finishing writing output for hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125320 so closing down at 1732726125643 (+23 ms)3 split writer threads finished at 1732726125643Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000000001-wal.1732726125320.temp (wrote 3000 edits, skipped 0 edits in 53 ms) at 1732726125653 (+10 ms)Rename recovered edits hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000000001-wal.1732726125320.temp to hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000003000 at 1732726125655 (+2 ms)Processed 3000 edits across 1 Regions in 78 ms; skipped=0; WAL=hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125320, size=257.2 K, length=263359, corrupted=false, cancelled=false at 1732726125655 2024-11-27T16:48:45,657 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125320 to hdfs://localhost:40409/hbase/oldWALs/wal.1732726125320 2024-11-27T16:48:45,658 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000003000 2024-11-27T16:48:45,658 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:48:45,661 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:45,677 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125661, exclude list is [], retry=0 2024-11-27T16:48:45,681 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:45,681 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:45,682 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:45,684 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125661 2024-11-27T16:48:45,688 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:33829:33829)] 2024-11-27T16:48:45,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741868_1045 (size=263486) 2024-11-27T16:48:45,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741868_1045 (size=263486) 2024-11-27T16:48:45,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741868_1045 (size=263486) 2024-11-27T16:48:45,859 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125661, size=257.3 K (263486bytes) 2024-11-27T16:48:45,859 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125661 2024-11-27T16:48:45,860 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125661 after 1ms 2024-11-27T16:48:45,863 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125661: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:45,865 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125661 took 6ms 2024-11-27T16:48:45,869 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732726125661.temp 2024-11-27T16:48:45,872 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000003001-wal.1732726125661.temp 2024-11-27T16:48:45,935 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125661 so closing down 2024-11-27T16:48:45,936 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T16:48:45,938 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T16:48:45,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741869_1046 (size=263486) 2024-11-27T16:48:45,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741869_1046 (size=263486) 2024-11-27T16:48:45,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741869_1046 (size=263486) 2024-11-27T16:48:45,944 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000003001-wal.1732726125661.temp (wrote 3000 edits, skipped 0 edits in 43 ms) 2024-11-27T16:48:45,946 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000003001-wal.1732726125661.temp to hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000006000 2024-11-27T16:48:45,946 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 81 ms; skipped=0; WAL=hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125661, size=257.3 K, length=263486, corrupted=false, cancelled=false 2024-11-27T16:48:45,946 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125661, journal: Splitting hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125661, size=257.3 K (263486bytes) at 1732726125859Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000003001-wal.1732726125661.temp at 1732726125872 (+13 ms)Split 1024 edits, skipped 0 edits. at 1732726125890 (+18 ms)Split 2048 edits, skipped 0 edits. at 1732726125913 (+23 ms)Finishing writing output for hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125661 so closing down at 1732726125936 (+23 ms)3 split writer threads finished at 1732726125938 (+2 ms)Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000003001-wal.1732726125661.temp (wrote 3000 edits, skipped 0 edits in 43 ms) at 1732726125944 (+6 ms)Rename recovered edits hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000003001-wal.1732726125661.temp to hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000006000 at 1732726125946 (+2 ms)Processed 3000 edits across 1 Regions in 81 ms; skipped=0; WAL=hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125661, size=257.3 K, length=263486, corrupted=false, cancelled=false at 1732726125946 2024-11-27T16:48:45,948 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125661 to hdfs://localhost:40409/hbase/oldWALs/wal.1732726125661 2024-11-27T16:48:45,949 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000006000 2024-11-27T16:48:45,949 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:48:45,952 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/test2727-manual,16010,1732726125247, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:45,968 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125952, exclude list is [], retry=0 2024-11-27T16:48:45,972 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:45,972 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:45,972 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:45,976 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732726125247/wal.1732726125952 2024-11-27T16:48:45,977 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:33829:33829)] 2024-11-27T16:48:45,977 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b22ea80fe1fb41359a4dc815a485075, NAME => 'test2727,,1732726125248.4b22ea80fe1fb41359a4dc815a485075.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:48:45,977 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732726125248.4b22ea80fe1fb41359a4dc815a485075.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:45,977 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:45,977 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:45,980 INFO [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:45,981 INFO [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b22ea80fe1fb41359a4dc815a485075 columnFamilyName a 2024-11-27T16:48:45,981 DEBUG [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:45,982 INFO [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] regionserver.HStore(327): Store=4b22ea80fe1fb41359a4dc815a485075/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:45,982 INFO [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:45,983 INFO [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b22ea80fe1fb41359a4dc815a485075 columnFamilyName b 2024-11-27T16:48:45,983 DEBUG [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:45,984 INFO [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] regionserver.HStore(327): Store=4b22ea80fe1fb41359a4dc815a485075/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:45,984 INFO [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:45,985 INFO [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b22ea80fe1fb41359a4dc815a485075 columnFamilyName c 2024-11-27T16:48:45,986 DEBUG [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:45,986 INFO [StoreOpener-4b22ea80fe1fb41359a4dc815a485075-1 {}] regionserver.HStore(327): Store=4b22ea80fe1fb41359a4dc815a485075/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:45,986 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:45,987 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:45,990 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:45,991 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000003000 2024-11-27T16:48:45,994 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:46,044 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-27T16:48:46,045 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-27T16:48:46,046 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T16:48:46,046 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-27T16:48:46,047 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-27T16:48:46,047 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-27T16:48:46,087 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-27T16:48:46,087 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-27T16:48:46,101 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000003000 2024-11-27T16:48:46,102 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000006000 2024-11-27T16:48:46,106 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:46,181 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000006000 2024-11-27T16:48:46,181 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4b22ea80fe1fb41359a4dc815a485075 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-27T16:48:46,214 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/.tmp/a/087ac6b4db62402d82e0b67a34407dfa is 41, key is test2727/a:100/1732726125692/Put/seqid=0 2024-11-27T16:48:46,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741871_1048 (size=84227) 2024-11-27T16:48:46,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741871_1048 (size=84227) 2024-11-27T16:48:46,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741871_1048 (size=84227) 2024-11-27T16:48:46,226 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/.tmp/a/087ac6b4db62402d82e0b67a34407dfa 2024-11-27T16:48:46,258 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/.tmp/b/0e0d03e3873c44a8b1d3e0d0439daba0 is 41, key is test2727/b:100/1732726125739/Put/seqid=0 2024-11-27T16:48:46,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741872_1049 (size=84609) 2024-11-27T16:48:46,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741872_1049 (size=84609) 2024-11-27T16:48:46,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741872_1049 (size=84609) 2024-11-27T16:48:46,270 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/.tmp/b/0e0d03e3873c44a8b1d3e0d0439daba0 2024-11-27T16:48:46,306 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/.tmp/c/090a1601ef3f457b923bfc29cf40f124 is 41, key is test2727/c:100/1732726125786/Put/seqid=0 2024-11-27T16:48:46,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741873_1050 (size=84609) 2024-11-27T16:48:46,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741873_1050 (size=84609) 2024-11-27T16:48:46,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741873_1050 (size=84609) 2024-11-27T16:48:46,314 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/.tmp/c/090a1601ef3f457b923bfc29cf40f124 2024-11-27T16:48:46,321 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/.tmp/a/087ac6b4db62402d82e0b67a34407dfa as hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/a/087ac6b4db62402d82e0b67a34407dfa 2024-11-27T16:48:46,327 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/a/087ac6b4db62402d82e0b67a34407dfa, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-27T16:48:46,329 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/.tmp/b/0e0d03e3873c44a8b1d3e0d0439daba0 as hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/b/0e0d03e3873c44a8b1d3e0d0439daba0 2024-11-27T16:48:46,336 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/b/0e0d03e3873c44a8b1d3e0d0439daba0, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-27T16:48:46,337 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/.tmp/c/090a1601ef3f457b923bfc29cf40f124 as hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/c/090a1601ef3f457b923bfc29cf40f124 2024-11-27T16:48:46,343 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/c/090a1601ef3f457b923bfc29cf40f124, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-27T16:48:46,343 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 4b22ea80fe1fb41359a4dc815a485075 in 162ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-27T16:48:46,344 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000003000 2024-11-27T16:48:46,344 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/0000000000000006000 2024-11-27T16:48:46,345 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:46,345 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:46,346 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T16:48:46,347 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4b22ea80fe1fb41359a4dc815a485075 2024-11-27T16:48:46,350 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/test2727/4b22ea80fe1fb41359a4dc815a485075/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-27T16:48:46,351 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4b22ea80fe1fb41359a4dc815a485075; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63470296, jitterRate=-0.05421888828277588}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T16:48:46,352 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4b22ea80fe1fb41359a4dc815a485075: Writing region info on filesystem at 1732726125978Initializing all the Stores at 1732726125979 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726125979Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726125979Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726125979Obtaining lock to block concurrent updates at 1732726126181 (+202 ms)Preparing flush snapshotting stores in 4b22ea80fe1fb41359a4dc815a485075 at 1732726126181Finished memstore snapshotting test2727,,1732726125248.4b22ea80fe1fb41359a4dc815a485075., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732726126181Flushing stores of test2727,,1732726125248.4b22ea80fe1fb41359a4dc815a485075. at 1732726126181Flushing 4b22ea80fe1fb41359a4dc815a485075/a: creating writer at 1732726126182 (+1 ms)Flushing 4b22ea80fe1fb41359a4dc815a485075/a: appending metadata at 1732726126213 (+31 ms)Flushing 4b22ea80fe1fb41359a4dc815a485075/a: closing flushed file at 1732726126213Flushing 4b22ea80fe1fb41359a4dc815a485075/b: creating writer at 1732726126233 (+20 ms)Flushing 4b22ea80fe1fb41359a4dc815a485075/b: appending metadata at 1732726126256 (+23 ms)Flushing 4b22ea80fe1fb41359a4dc815a485075/b: closing flushed file at 1732726126256Flushing 4b22ea80fe1fb41359a4dc815a485075/c: creating writer at 1732726126277 (+21 ms)Flushing 4b22ea80fe1fb41359a4dc815a485075/c: appending metadata at 1732726126305 (+28 ms)Flushing 4b22ea80fe1fb41359a4dc815a485075/c: closing flushed file at 1732726126305Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4218e9c1: reopening flushed file at 1732726126320 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@363a231f: reopening flushed file at 1732726126328 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7108bdd2: reopening flushed file at 1732726126336 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 4b22ea80fe1fb41359a4dc815a485075 in 162ms, sequenceid=6000, compaction requested=false; wal=null at 1732726126343 (+7 ms)Cleaning up temporary data from old regions at 1732726126345 (+2 ms)Region opened successfully at 1732726126352 (+7 ms) 2024-11-27T16:48:46,353 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-27T16:48:46,353 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4b22ea80fe1fb41359a4dc815a485075, disabling compactions & flushes 2024-11-27T16:48:46,353 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732726125248.4b22ea80fe1fb41359a4dc815a485075. 2024-11-27T16:48:46,353 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732726125248.4b22ea80fe1fb41359a4dc815a485075. 2024-11-27T16:48:46,353 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732726125248.4b22ea80fe1fb41359a4dc815a485075. after waiting 0 ms 2024-11-27T16:48:46,353 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732726125248.4b22ea80fe1fb41359a4dc815a485075. 2024-11-27T16:48:46,355 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732726125248.4b22ea80fe1fb41359a4dc815a485075. 2024-11-27T16:48:46,355 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4b22ea80fe1fb41359a4dc815a485075: Waiting for close lock at 1732726126353Disabling compacts and flushes for region at 1732726126353Disabling writes for close at 1732726126353Writing region close event to WAL at 1732726126355 (+2 ms)Closed at 1732726126355 2024-11-27T16:48:46,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741870_1047 (size=95) 2024-11-27T16:48:46,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741870_1047 (size=95) 2024-11-27T16:48:46,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741870_1047 (size=95) 2024-11-27T16:48:46,362 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T16:48:46,362 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732726125952) 2024-11-27T16:48:46,380 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=405 (was 403) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:50304 [Waiting for operation #28] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:46394 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:35248 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=250 (was 250), ProcessCount=11 (was 11), AvailableMemoryMB=6599 (was 6843) 2024-11-27T16:48:46,391 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=405, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=250, ProcessCount=11, AvailableMemoryMB=6599 2024-11-27T16:48:46,410 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:46,418 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732726126409, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:46,419 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732726126419 2024-11-27T16:48:46,431 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732726126409/wal.1732726126419 2024-11-27T16:48:46,435 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:33829:33829)] 2024-11-27T16:48:46,440 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ef866390e5e753f52b913a04e580b4da, NAME => 'testSequentialEditLogSeqNum,,1732726126411.ef866390e5e753f52b913a04e580b4da.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:48:46,440 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732726126411.ef866390e5e753f52b913a04e580b4da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:46,440 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ef866390e5e753f52b913a04e580b4da 2024-11-27T16:48:46,440 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ef866390e5e753f52b913a04e580b4da 2024-11-27T16:48:46,441 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:40409/hbase/data/default/testSequentialEditLogSeqNum/ef866390e5e753f52b913a04e580b4da doesn't exist for region: ef866390e5e753f52b913a04e580b4da on table testSequentialEditLogSeqNum 2024-11-27T16:48:46,441 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: ef866390e5e753f52b913a04e580b4da on table testSequentialEditLogSeqNum 2024-11-27T16:48:46,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741875_1052 (size=62) 2024-11-27T16:48:46,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741875_1052 (size=62) 2024-11-27T16:48:46,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741875_1052 (size=62) 2024-11-27T16:48:46,462 INFO [StoreOpener-ef866390e5e753f52b913a04e580b4da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ef866390e5e753f52b913a04e580b4da 2024-11-27T16:48:46,463 INFO [StoreOpener-ef866390e5e753f52b913a04e580b4da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef866390e5e753f52b913a04e580b4da columnFamilyName a 2024-11-27T16:48:46,464 DEBUG [StoreOpener-ef866390e5e753f52b913a04e580b4da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:46,464 INFO [StoreOpener-ef866390e5e753f52b913a04e580b4da-1 {}] regionserver.HStore(327): Store=ef866390e5e753f52b913a04e580b4da/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:46,464 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ef866390e5e753f52b913a04e580b4da 2024-11-27T16:48:46,465 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testSequentialEditLogSeqNum/ef866390e5e753f52b913a04e580b4da 2024-11-27T16:48:46,465 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testSequentialEditLogSeqNum/ef866390e5e753f52b913a04e580b4da 2024-11-27T16:48:46,466 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ef866390e5e753f52b913a04e580b4da 2024-11-27T16:48:46,466 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ef866390e5e753f52b913a04e580b4da 2024-11-27T16:48:46,468 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ef866390e5e753f52b913a04e580b4da 2024-11-27T16:48:46,470 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testSequentialEditLogSeqNum/ef866390e5e753f52b913a04e580b4da/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T16:48:46,471 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ef866390e5e753f52b913a04e580b4da; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71702589, jitterRate=0.06845183670520782}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T16:48:46,471 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ef866390e5e753f52b913a04e580b4da: Writing region info on filesystem at 1732726126440Initializing all the Stores at 1732726126461 (+21 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726126461Cleaning up temporary data from old regions at 1732726126466 (+5 ms)Region opened successfully at 1732726126471 (+5 ms) 2024-11-27T16:48:46,484 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ef866390e5e753f52b913a04e580b4da 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-27T16:48:46,507 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testSequentialEditLogSeqNum/ef866390e5e753f52b913a04e580b4da/.tmp/a/da006ab1aa8d428294417ec26ef31fae is 81, key is testSequentialEditLogSeqNum/a:x0/1732726126472/Put/seqid=0 2024-11-27T16:48:46,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741876_1053 (size=5833) 2024-11-27T16:48:46,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741876_1053 (size=5833) 2024-11-27T16:48:46,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741876_1053 (size=5833) 2024-11-27T16:48:46,518 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testSequentialEditLogSeqNum/ef866390e5e753f52b913a04e580b4da/.tmp/a/da006ab1aa8d428294417ec26ef31fae 2024-11-27T16:48:46,527 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testSequentialEditLogSeqNum/ef866390e5e753f52b913a04e580b4da/.tmp/a/da006ab1aa8d428294417ec26ef31fae as hdfs://localhost:40409/hbase/data/default/testSequentialEditLogSeqNum/ef866390e5e753f52b913a04e580b4da/a/da006ab1aa8d428294417ec26ef31fae 2024-11-27T16:48:46,533 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testSequentialEditLogSeqNum/ef866390e5e753f52b913a04e580b4da/a/da006ab1aa8d428294417ec26ef31fae, entries=10, sequenceid=13, filesize=5.7 K 2024-11-27T16:48:46,534 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for ef866390e5e753f52b913a04e580b4da in 50ms, sequenceid=13, compaction requested=false 2024-11-27T16:48:46,535 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ef866390e5e753f52b913a04e580b4da: 2024-11-27T16:48:46,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T16:48:46,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T16:48:46,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T16:48:46,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T16:48:46,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T16:48:46,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741874_1051 (size=1843) 2024-11-27T16:48:46,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741874_1051 (size=1843) 2024-11-27T16:48:46,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741874_1051 (size=1843) 2024-11-27T16:48:46,571 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40409/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732726126409/wal.1732726126419, size=1.8 K (1843bytes) 2024-11-27T16:48:46,572 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40409/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732726126409/wal.1732726126419 2024-11-27T16:48:46,572 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40409/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732726126409/wal.1732726126419 after 0ms 2024-11-27T16:48:46,575 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732726126409/wal.1732726126419: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:46,576 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40409/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732726126409/wal.1732726126419 took 5ms 2024-11-27T16:48:46,579 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40409/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732726126409/wal.1732726126419 so closing down 2024-11-27T16:48:46,579 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T16:48:46,580 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732726126419.temp 2024-11-27T16:48:46,581 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testSequentialEditLogSeqNum/ef866390e5e753f52b913a04e580b4da/recovered.edits/0000000000000000003-wal.1732726126419.temp 2024-11-27T16:48:46,582 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T16:48:46,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741877_1054 (size=1477) 2024-11-27T16:48:46,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741877_1054 (size=1477) 2024-11-27T16:48:46,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741877_1054 (size=1477) 2024-11-27T16:48:46,591 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testSequentialEditLogSeqNum/ef866390e5e753f52b913a04e580b4da/recovered.edits/0000000000000000003-wal.1732726126419.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-27T16:48:46,593 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40409/hbase/data/default/testSequentialEditLogSeqNum/ef866390e5e753f52b913a04e580b4da/recovered.edits/0000000000000000003-wal.1732726126419.temp to hdfs://localhost:40409/hbase/data/default/testSequentialEditLogSeqNum/ef866390e5e753f52b913a04e580b4da/recovered.edits/0000000000000000020 2024-11-27T16:48:46,593 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 17 ms; skipped=2; WAL=hdfs://localhost:40409/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732726126409/wal.1732726126419, size=1.8 K, length=1843, corrupted=false, cancelled=false 2024-11-27T16:48:46,593 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40409/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732726126409/wal.1732726126419, journal: Splitting hdfs://localhost:40409/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732726126409/wal.1732726126419, size=1.8 K (1843bytes) at 1732726126571Finishing writing output for hdfs://localhost:40409/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732726126409/wal.1732726126419 so closing down at 1732726126579 (+8 ms)Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testSequentialEditLogSeqNum/ef866390e5e753f52b913a04e580b4da/recovered.edits/0000000000000000003-wal.1732726126419.temp at 1732726126581 (+2 ms)3 split writer threads finished at 1732726126582 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testSequentialEditLogSeqNum/ef866390e5e753f52b913a04e580b4da/recovered.edits/0000000000000000003-wal.1732726126419.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732726126591 (+9 ms)Rename recovered edits hdfs://localhost:40409/hbase/data/default/testSequentialEditLogSeqNum/ef866390e5e753f52b913a04e580b4da/recovered.edits/0000000000000000003-wal.1732726126419.temp to hdfs://localhost:40409/hbase/data/default/testSequentialEditLogSeqNum/ef866390e5e753f52b913a04e580b4da/recovered.edits/0000000000000000020 at 1732726126593 (+2 ms)Processed 17 edits across 1 Regions in 17 ms; skipped=2; WAL=hdfs://localhost:40409/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732726126409/wal.1732726126419, size=1.8 K, length=1843, corrupted=false, cancelled=false at 1732726126593 2024-11-27T16:48:46,612 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=410 (was 405) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:50304 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:35248 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=250 (was 250), ProcessCount=11 (was 11), AvailableMemoryMB=6598 (was 6599) 2024-11-27T16:48:46,626 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=410, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=250, ProcessCount=11, AvailableMemoryMB=6597 2024-11-27T16:48:46,650 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:46,653 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:46,688 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T16:48:46,692 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-41025511, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/hregion-41025511, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:46,706 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-41025511/hregion-41025511.1732726126692, exclude list is [], retry=0 2024-11-27T16:48:46,710 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:46,711 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:46,711 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:46,719 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-41025511/hregion-41025511.1732726126692 2024-11-27T16:48:46,719 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:39655:39655)] 2024-11-27T16:48:46,719 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7d112e23cae468613a1a89004b8bc875, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732726126651.7d112e23cae468613a1a89004b8bc875.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40409/hbase 2024-11-27T16:48:46,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741879_1056 (size=70) 2024-11-27T16:48:46,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741879_1056 (size=70) 2024-11-27T16:48:46,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741879_1056 (size=70) 2024-11-27T16:48:46,737 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732726126651.7d112e23cae468613a1a89004b8bc875.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:46,739 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,740 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7d112e23cae468613a1a89004b8bc875 columnFamilyName a 2024-11-27T16:48:46,740 DEBUG [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:46,741 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] regionserver.HStore(327): Store=7d112e23cae468613a1a89004b8bc875/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:46,741 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,742 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7d112e23cae468613a1a89004b8bc875 columnFamilyName b 2024-11-27T16:48:46,742 DEBUG [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:46,743 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] regionserver.HStore(327): Store=7d112e23cae468613a1a89004b8bc875/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:46,743 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,744 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7d112e23cae468613a1a89004b8bc875 columnFamilyName c 2024-11-27T16:48:46,744 DEBUG [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:46,745 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] regionserver.HStore(327): Store=7d112e23cae468613a1a89004b8bc875/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:46,745 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,745 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,746 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,747 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,747 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,747 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T16:48:46,749 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,751 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T16:48:46,751 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7d112e23cae468613a1a89004b8bc875; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72270315, jitterRate=0.07691161334514618}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T16:48:46,752 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7d112e23cae468613a1a89004b8bc875: Writing region info on filesystem at 1732726126737Initializing all the Stores at 1732726126738 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726126738Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726126738Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726126738Cleaning up temporary data from old regions at 1732726126747 (+9 ms)Region opened successfully at 1732726126752 (+5 ms) 2024-11-27T16:48:46,752 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7d112e23cae468613a1a89004b8bc875, disabling compactions & flushes 2024-11-27T16:48:46,752 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732726126651.7d112e23cae468613a1a89004b8bc875. 2024-11-27T16:48:46,752 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732726126651.7d112e23cae468613a1a89004b8bc875. 2024-11-27T16:48:46,752 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732726126651.7d112e23cae468613a1a89004b8bc875. after waiting 0 ms 2024-11-27T16:48:46,752 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732726126651.7d112e23cae468613a1a89004b8bc875. 2024-11-27T16:48:46,753 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732726126651.7d112e23cae468613a1a89004b8bc875. 2024-11-27T16:48:46,753 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7d112e23cae468613a1a89004b8bc875: Waiting for close lock at 1732726126752Disabling compacts and flushes for region at 1732726126752Disabling writes for close at 1732726126752Writing region close event to WAL at 1732726126753 (+1 ms)Closed at 1732726126753 2024-11-27T16:48:46,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741878_1055 (size=95) 2024-11-27T16:48:46,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741878_1055 (size=95) 2024-11-27T16:48:46,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741878_1055 (size=95) 2024-11-27T16:48:46,758 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T16:48:46,758 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-41025511:(num 1732726126692) 2024-11-27T16:48:46,759 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:48:46,761 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:46,776 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726126761, exclude list is [], retry=0 2024-11-27T16:48:46,779 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:46,780 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:46,780 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:46,783 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726126761 2024-11-27T16:48:46,783 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:39655:39655)] 2024-11-27T16:48:46,783 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7d112e23cae468613a1a89004b8bc875, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732726126651.7d112e23cae468613a1a89004b8bc875.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:48:46,783 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732726126651.7d112e23cae468613a1a89004b8bc875.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:46,783 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,783 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,785 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,786 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7d112e23cae468613a1a89004b8bc875 columnFamilyName a 2024-11-27T16:48:46,786 DEBUG [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:46,787 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] regionserver.HStore(327): Store=7d112e23cae468613a1a89004b8bc875/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:46,787 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,788 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7d112e23cae468613a1a89004b8bc875 columnFamilyName b 2024-11-27T16:48:46,788 DEBUG [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:46,788 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] regionserver.HStore(327): Store=7d112e23cae468613a1a89004b8bc875/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:46,788 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,789 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7d112e23cae468613a1a89004b8bc875 columnFamilyName c 2024-11-27T16:48:46,789 DEBUG [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:46,790 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] regionserver.HStore(327): Store=7d112e23cae468613a1a89004b8bc875/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:46,790 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,791 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,792 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,793 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,793 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,793 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T16:48:46,795 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:46,795 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7d112e23cae468613a1a89004b8bc875; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74756652, jitterRate=0.11396092176437378}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T16:48:46,797 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7d112e23cae468613a1a89004b8bc875: Writing region info on filesystem at 1732726126784Initializing all the Stores at 1732726126784Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726126785 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726126785Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726126785Cleaning up temporary data from old regions at 1732726126793 (+8 ms)Region opened successfully at 1732726126797 (+4 ms) 2024-11-27T16:48:46,801 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732726126800/Put/seqid=0 2024-11-27T16:48:46,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741881_1058 (size=4826) 2024-11-27T16:48:46,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741881_1058 (size=4826) 2024-11-27T16:48:46,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741881_1058 (size=4826) 2024-11-27T16:48:46,816 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40409/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 7d112e23cae468613a1a89004b8bc875/a 2024-11-27T16:48:46,827 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-27T16:48:46,827 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-27T16:48:46,827 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7d112e23cae468613a1a89004b8bc875: 2024-11-27T16:48:46,829 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/a/c0c3647cac1f4df2aff0d9889f0acd17_SeqId_3_ 2024-11-27T16:48:46,830 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40409/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 7d112e23cae468613a1a89004b8bc875/a as hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/a/c0c3647cac1f4df2aff0d9889f0acd17_SeqId_3_ - updating store file list. 2024-11-27T16:48:46,836 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for c0c3647cac1f4df2aff0d9889f0acd17_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-27T16:48:46,837 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/a/c0c3647cac1f4df2aff0d9889f0acd17_SeqId_3_ into 7d112e23cae468613a1a89004b8bc875/a 2024-11-27T16:48:46,837 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40409/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 7d112e23cae468613a1a89004b8bc875/a (new location: hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/a/c0c3647cac1f4df2aff0d9889f0acd17_SeqId_3_) 2024-11-27T16:48:46,892 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40409/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726126761, size=0 (0bytes) 2024-11-27T16:48:46,892 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40409/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726126761 might be still open, length is 0 2024-11-27T16:48:46,892 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40409/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726126761 2024-11-27T16:48:46,893 WARN [IPC Server handler 4 on default port 40409 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726126761 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-27T16:48:46,893 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40409/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726126761 after 1ms 2024-11-27T16:48:49,465 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:35324 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41739:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35324 dst: /127.0.0.1:41739 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41739 remote=/127.0.0.1:35324]. Total timeout mills is 60000, 57379 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:48:49,466 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:46466 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37511:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46466 dst: /127.0.0.1:37511 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:48:49,466 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:50520 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37433:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50520 dst: /127.0.0.1:37433 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:48:49,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741880_1059 (size=473) 2024-11-27T16:48:49,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741880_1059 (size=473) 2024-11-27T16:48:49,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741880_1059 (size=473) 2024-11-27T16:48:50,894 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40409/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726126761 after 4002ms 2024-11-27T16:48:50,897 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726126761: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:50,898 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40409/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726126761 took 4006ms 2024-11-27T16:48:50,900 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40409/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726126761; continuing. 2024-11-27T16:48:50,900 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40409/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726126761 so closing down 2024-11-27T16:48:50,900 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T16:48:50,902 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732726126761.temp 2024-11-27T16:48:50,903 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/recovered.edits/0000000000000000005-wal.1732726126761.temp 2024-11-27T16:48:50,904 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T16:48:50,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741882_1060 (size=259) 2024-11-27T16:48:50,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741882_1060 (size=259) 2024-11-27T16:48:50,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741882_1060 (size=259) 2024-11-27T16:48:50,919 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/recovered.edits/0000000000000000005-wal.1732726126761.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-27T16:48:50,921 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/recovered.edits/0000000000000000005-wal.1732726126761.temp to hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/recovered.edits/0000000000000000005 2024-11-27T16:48:50,921 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 23 ms; skipped=1; WAL=hdfs://localhost:40409/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726126761, size=0, length=0, corrupted=false, cancelled=false 2024-11-27T16:48:50,921 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40409/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726126761, journal: Splitting hdfs://localhost:40409/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726126761, size=0 (0bytes) at 1732726126892Finishing writing output for hdfs://localhost:40409/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726126761 so closing down at 1732726130900 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/recovered.edits/0000000000000000005-wal.1732726126761.temp at 1732726130903 (+3 ms)3 split writer threads finished at 1732726130904 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/recovered.edits/0000000000000000005-wal.1732726126761.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732726130919 (+15 ms)Rename recovered edits hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/recovered.edits/0000000000000000005-wal.1732726126761.temp to hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/recovered.edits/0000000000000000005 at 1732726130921 (+2 ms)Processed 2 edits across 1 Regions in 23 ms; skipped=1; WAL=hdfs://localhost:40409/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726126761, size=0, length=0, corrupted=false, cancelled=false at 1732726130921 2024-11-27T16:48:50,923 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40409/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726126761 to hdfs://localhost:40409/hbase/oldWALs/wal.1732726126761 2024-11-27T16:48:50,923 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/recovered.edits/0000000000000000005 2024-11-27T16:48:50,924 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:48:50,926 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:50,939 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726130926, exclude list is [], retry=0 2024-11-27T16:48:50,942 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:50,942 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:50,943 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:50,945 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726130926 2024-11-27T16:48:50,945 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:42883:42883)] 2024-11-27T16:48:50,945 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7d112e23cae468613a1a89004b8bc875, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732726126651.7d112e23cae468613a1a89004b8bc875.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:48:50,945 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732726126651.7d112e23cae468613a1a89004b8bc875.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:50,945 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:50,945 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:50,947 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:50,948 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7d112e23cae468613a1a89004b8bc875 columnFamilyName a 2024-11-27T16:48:50,948 DEBUG [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:50,954 DEBUG [StoreFileOpener-7d112e23cae468613a1a89004b8bc875-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for c0c3647cac1f4df2aff0d9889f0acd17_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-27T16:48:50,955 DEBUG [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/a/c0c3647cac1f4df2aff0d9889f0acd17_SeqId_3_ 2024-11-27T16:48:50,955 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] regionserver.HStore(327): Store=7d112e23cae468613a1a89004b8bc875/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:50,955 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:50,956 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7d112e23cae468613a1a89004b8bc875 columnFamilyName b 2024-11-27T16:48:50,956 DEBUG [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:50,956 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] regionserver.HStore(327): Store=7d112e23cae468613a1a89004b8bc875/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:50,956 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:50,957 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7d112e23cae468613a1a89004b8bc875 columnFamilyName c 2024-11-27T16:48:50,958 DEBUG [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:50,958 INFO [StoreOpener-7d112e23cae468613a1a89004b8bc875-1 {}] regionserver.HStore(327): Store=7d112e23cae468613a1a89004b8bc875/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:50,958 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:50,959 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:50,960 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:50,961 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/recovered.edits/0000000000000000005 2024-11-27T16:48:50,963 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:50,964 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/recovered.edits/0000000000000000005 2024-11-27T16:48:50,965 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7d112e23cae468613a1a89004b8bc875 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-27T16:48:50,980 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/.tmp/a/d2a79ccc2df74034bc404aaae01d4de1 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732726126843/Put/seqid=0 2024-11-27T16:48:50,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741884_1062 (size=5149) 2024-11-27T16:48:50,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741884_1062 (size=5149) 2024-11-27T16:48:50,988 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/.tmp/a/d2a79ccc2df74034bc404aaae01d4de1 2024-11-27T16:48:50,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741884_1062 (size=5149) 2024-11-27T16:48:50,995 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/.tmp/a/d2a79ccc2df74034bc404aaae01d4de1 as hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/a/d2a79ccc2df74034bc404aaae01d4de1 2024-11-27T16:48:51,000 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/a/d2a79ccc2df74034bc404aaae01d4de1, entries=1, sequenceid=5, filesize=5.0 K 2024-11-27T16:48:51,001 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 7d112e23cae468613a1a89004b8bc875 in 36ms, sequenceid=5, compaction requested=false; wal=null 2024-11-27T16:48:51,001 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/recovered.edits/0000000000000000005 2024-11-27T16:48:51,003 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:51,003 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:51,003 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T16:48:51,005 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7d112e23cae468613a1a89004b8bc875 2024-11-27T16:48:51,008 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7d112e23cae468613a1a89004b8bc875/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-27T16:48:51,009 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7d112e23cae468613a1a89004b8bc875; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59238729, jitterRate=-0.11727415025234222}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T16:48:51,009 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7d112e23cae468613a1a89004b8bc875: Writing region info on filesystem at 1732726130946Initializing all the Stores at 1732726130946Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726130946Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726130947 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726130947Obtaining lock to block concurrent updates at 1732726130965 (+18 ms)Preparing flush snapshotting stores in 7d112e23cae468613a1a89004b8bc875 at 1732726130965Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732726126651.7d112e23cae468613a1a89004b8bc875., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732726130965Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732726126651.7d112e23cae468613a1a89004b8bc875. at 1732726130965Flushing 7d112e23cae468613a1a89004b8bc875/a: creating writer at 1732726130965Flushing 7d112e23cae468613a1a89004b8bc875/a: appending metadata at 1732726130979 (+14 ms)Flushing 7d112e23cae468613a1a89004b8bc875/a: closing flushed file at 1732726130979Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76ff9cf4: reopening flushed file at 1732726130994 (+15 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 7d112e23cae468613a1a89004b8bc875 in 36ms, sequenceid=5, compaction requested=false; wal=null at 1732726131001 (+7 ms)Cleaning up temporary data from old regions at 1732726131003 (+2 ms)Region opened successfully at 1732726131009 (+6 ms) 2024-11-27T16:48:51,013 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7d112e23cae468613a1a89004b8bc875, disabling compactions & flushes 2024-11-27T16:48:51,013 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732726126651.7d112e23cae468613a1a89004b8bc875. 2024-11-27T16:48:51,013 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732726126651.7d112e23cae468613a1a89004b8bc875. 2024-11-27T16:48:51,013 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732726126651.7d112e23cae468613a1a89004b8bc875. after waiting 0 ms 2024-11-27T16:48:51,013 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732726126651.7d112e23cae468613a1a89004b8bc875. 2024-11-27T16:48:51,014 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732726126651.7d112e23cae468613a1a89004b8bc875. 2024-11-27T16:48:51,014 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7d112e23cae468613a1a89004b8bc875: Waiting for close lock at 1732726131013Disabling compacts and flushes for region at 1732726131013Disabling writes for close at 1732726131013Writing region close event to WAL at 1732726131014 (+1 ms)Closed at 1732726131014 2024-11-27T16:48:51,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741883_1061 (size=95) 2024-11-27T16:48:51,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741883_1061 (size=95) 2024-11-27T16:48:51,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741883_1061 (size=95) 2024-11-27T16:48:51,020 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T16:48:51,020 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732726130926) 2024-11-27T16:48:51,037 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=412 (was 410) Potentially hanging thread: IPC Client (1112330208) connection to localhost/127.0.0.1:40409 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1471535619_22 at /127.0.0.1:35332 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:40409 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1471535619_22 at /127.0.0.1:50564 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1022 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=254 (was 250) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6600 (was 6597) - AvailableMemoryMB LEAK? - 2024-11-27T16:48:51,049 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=412, OpenFileDescriptor=1022, MaxFileDescriptor=1048576, SystemLoadAverage=254, ProcessCount=11, AvailableMemoryMB=6600 2024-11-27T16:48:51,064 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:51,070 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T16:48:51,074 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 37bfedfd9766,35959,1732726115618 2024-11-27T16:48:51,077 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@732c3191 2024-11-27T16:48:51,078 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T16:48:51,080 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T16:48:51,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35959 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T16:48:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35959 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-27T16:48:51,093 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T16:48:51,096 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35959 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-27T16:48:51,097 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:51,100 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T16:48:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T16:48:51,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741885_1063 (size=694) 2024-11-27T16:48:51,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741885_1063 (size=694) 2024-11-27T16:48:51,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741885_1063 (size=694) 2024-11-27T16:48:51,113 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e7d41a86458ce2565a8f7b7bb1771881, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4 2024-11-27T16:48:51,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741886_1064 (size=77) 2024-11-27T16:48:51,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741886_1064 (size=77) 2024-11-27T16:48:51,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741886_1064 (size=77) 2024-11-27T16:48:51,122 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:51,122 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing e7d41a86458ce2565a8f7b7bb1771881, disabling compactions & flushes 2024-11-27T16:48:51,123 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:51,123 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:51,123 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. after waiting 0 ms 2024-11-27T16:48:51,123 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:51,123 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:51,123 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for e7d41a86458ce2565a8f7b7bb1771881: Waiting for close lock at 1732726131122Disabling compacts and flushes for region at 1732726131122Disabling writes for close at 1732726131123 (+1 ms)Writing region close event to WAL at 1732726131123Closed at 1732726131123 2024-11-27T16:48:51,124 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T16:48:51,131 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732726131125"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732726131125"}]},"ts":"1732726131125"} 2024-11-27T16:48:51,135 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T16:48:51,136 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T16:48:51,139 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732726131136"}]},"ts":"1732726131136"} 2024-11-27T16:48:51,143 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-27T16:48:51,143 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {37bfedfd9766=0} racks are {/default-rack=0} 2024-11-27T16:48:51,144 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T16:48:51,145 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T16:48:51,145 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-27T16:48:51,145 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T16:48:51,145 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T16:48:51,145 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-27T16:48:51,145 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T16:48:51,145 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T16:48:51,145 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-27T16:48:51,145 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T16:48:51,146 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, ASSIGN}] 2024-11-27T16:48:51,148 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, ASSIGN 2024-11-27T16:48:51,149 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, ASSIGN; state=OFFLINE, location=37bfedfd9766,37003,1732726116450; forceNewPlan=false, retain=false 2024-11-27T16:48:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T16:48:51,302 INFO [37bfedfd9766:35959 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-27T16:48:51,303 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e7d41a86458ce2565a8f7b7bb1771881, regionState=OPENING, regionLocation=37bfedfd9766,37003,1732726116450 2024-11-27T16:48:51,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, ASSIGN because future has completed 2024-11-27T16:48:51,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7d41a86458ce2565a8f7b7bb1771881, server=37bfedfd9766,37003,1732726116450}] 2024-11-27T16:48:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T16:48:51,460 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T16:48:51,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44829, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T16:48:51,467 INFO [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:51,468 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e7d41a86458ce2565a8f7b7bb1771881, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:48:51,468 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:51,468 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:51,468 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:51,468 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:51,470 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:51,471 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7d41a86458ce2565a8f7b7bb1771881 columnFamilyName cf1 2024-11-27T16:48:51,471 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:51,472 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(327): Store=e7d41a86458ce2565a8f7b7bb1771881/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:51,472 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:51,473 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7d41a86458ce2565a8f7b7bb1771881 columnFamilyName cf2 2024-11-27T16:48:51,473 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:51,474 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(327): Store=e7d41a86458ce2565a8f7b7bb1771881/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:51,474 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:51,474 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:51,475 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:51,476 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:51,476 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:51,477 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-27T16:48:51,478 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:51,480 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T16:48:51,481 INFO [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e7d41a86458ce2565a8f7b7bb1771881; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62577435, jitterRate=-0.06752355396747589}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-27T16:48:51,481 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:51,481 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e7d41a86458ce2565a8f7b7bb1771881: Running coprocessor pre-open hook at 1732726131468Writing region info on filesystem at 1732726131468Initializing all the Stores at 1732726131469 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726131469Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726131470 (+1 ms)Cleaning up temporary data from old regions at 1732726131476 (+6 ms)Running coprocessor post-open hooks at 1732726131481 (+5 ms)Region opened successfully at 1732726131481 2024-11-27T16:48:51,483 INFO [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., pid=6, masterSystemTime=1732726131459 2024-11-27T16:48:51,486 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:51,486 INFO [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:51,487 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e7d41a86458ce2565a8f7b7bb1771881, regionState=OPEN, openSeqNum=2, regionLocation=37bfedfd9766,37003,1732726116450 2024-11-27T16:48:51,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7d41a86458ce2565a8f7b7bb1771881, server=37bfedfd9766,37003,1732726116450 because future has completed 2024-11-27T16:48:51,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-27T16:48:51,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e7d41a86458ce2565a8f7b7bb1771881, server=37bfedfd9766,37003,1732726116450 in 186 msec 2024-11-27T16:48:51,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-27T16:48:51,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, ASSIGN in 349 msec 2024-11-27T16:48:51,500 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T16:48:51,500 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732726131500"}]},"ts":"1732726131500"} 2024-11-27T16:48:51,507 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-27T16:48:51,509 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T16:48:51,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 423 msec 2024-11-27T16:48:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T16:48:51,733 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-27T16:48:51,733 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-27T16:48:51,735 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T16:48:51,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-27T16:48:51,741 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T16:48:51,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-27T16:48:51,755 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=2] 2024-11-27T16:48:51,756 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T16:48:51,758 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T16:48:51,773 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35959 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=e7d41a86458ce2565a8f7b7bb1771881, source=37bfedfd9766,37003,1732726116450, destination=37bfedfd9766,40989,1732726116343, warming up region on 37bfedfd9766,40989,1732726116343 2024-11-27T16:48:51,775 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35959 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=e7d41a86458ce2565a8f7b7bb1771881, source=37bfedfd9766,37003,1732726116450, destination=37bfedfd9766,40989,1732726116343, running balancer 2024-11-27T16:48:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35959 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, REOPEN/MOVE 2024-11-27T16:48:51,777 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, REOPEN/MOVE 2024-11-27T16:48:51,780 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.HRegion(7855): Warmup {ENCODED => e7d41a86458ce2565a8f7b7bb1771881, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:48:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:51,781 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e7d41a86458ce2565a8f7b7bb1771881, regionState=CLOSING, regionLocation=37bfedfd9766,37003,1732726116450 2024-11-27T16:48:51,784 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:51,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, REOPEN/MOVE because future has completed 2024-11-27T16:48:51,786 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7d41a86458ce2565a8f7b7bb1771881 columnFamilyName cf1 2024-11-27T16:48:51,786 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T16:48:51,786 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:51,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7d41a86458ce2565a8f7b7bb1771881, server=37bfedfd9766,37003,1732726116450}] 2024-11-27T16:48:51,786 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(327): Store=e7d41a86458ce2565a8f7b7bb1771881/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:51,787 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:51,789 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7d41a86458ce2565a8f7b7bb1771881 columnFamilyName cf2 2024-11-27T16:48:51,789 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:51,789 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(327): Store=e7d41a86458ce2565a8f7b7bb1771881/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.HRegion(1722): Closing e7d41a86458ce2565a8f7b7bb1771881, disabling compactions & flushes 2024-11-27T16:48:51,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. after waiting 0 ms 2024-11-27T16:48:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:51,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.HRegion(1676): Region close journal for e7d41a86458ce2565a8f7b7bb1771881: Waiting for close lock at 1732726131789Disabling compacts and flushes for region at 1732726131789Disabling writes for close at 1732726131790 (+1 ms)Writing region close event to WAL at 1732726131790Closed at 1732726131790 2024-11-27T16:48:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35959 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-27T16:48:51,946 INFO [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:51,946 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T16:48:51,947 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing e7d41a86458ce2565a8f7b7bb1771881, disabling compactions & flushes 2024-11-27T16:48:51,947 INFO [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:51,947 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:51,947 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. after waiting 0 ms 2024-11-27T16:48:51,947 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:51,947 INFO [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing e7d41a86458ce2565a8f7b7bb1771881 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-27T16:48:51,966 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/.tmp/cf1/7bb6af69c06d4acb8c822ea1fea2f012 is 35, key is r1/cf1:q/1732726131759/Put/seqid=0 2024-11-27T16:48:51,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741887_1065 (size=4783) 2024-11-27T16:48:51,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741887_1065 (size=4783) 2024-11-27T16:48:51,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741887_1065 (size=4783) 2024-11-27T16:48:51,976 INFO [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/.tmp/cf1/7bb6af69c06d4acb8c822ea1fea2f012 2024-11-27T16:48:51,992 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/.tmp/cf1/7bb6af69c06d4acb8c822ea1fea2f012 as hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf1/7bb6af69c06d4acb8c822ea1fea2f012 2024-11-27T16:48:51,999 INFO [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf1/7bb6af69c06d4acb8c822ea1fea2f012, entries=1, sequenceid=5, filesize=4.7 K 2024-11-27T16:48:52,001 INFO [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for e7d41a86458ce2565a8f7b7bb1771881 in 54ms, sequenceid=5, compaction requested=false 2024-11-27T16:48:52,001 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-27T16:48:52,007 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-27T16:48:52,010 INFO [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:52,010 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for e7d41a86458ce2565a8f7b7bb1771881: Waiting for close lock at 1732726131947Running coprocessor pre-close hooks at 1732726131947Disabling compacts and flushes for region at 1732726131947Disabling writes for close at 1732726131947Obtaining lock to block concurrent updates at 1732726131947Preparing flush snapshotting stores in e7d41a86458ce2565a8f7b7bb1771881 at 1732726131947Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732726131947Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. at 1732726131948 (+1 ms)Flushing e7d41a86458ce2565a8f7b7bb1771881/cf1: creating writer at 1732726131948Flushing e7d41a86458ce2565a8f7b7bb1771881/cf1: appending metadata at 1732726131965 (+17 ms)Flushing e7d41a86458ce2565a8f7b7bb1771881/cf1: closing flushed file at 1732726131965Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51e2dc07: reopening flushed file at 1732726131991 (+26 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for e7d41a86458ce2565a8f7b7bb1771881 in 54ms, sequenceid=5, compaction requested=false at 1732726132001 (+10 ms)Writing region close event to WAL at 1732726132003 (+2 ms)Running coprocessor post-close hooks at 1732726132007 (+4 ms)Closed at 1732726132010 (+3 ms) 2024-11-27T16:48:52,010 INFO [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding e7d41a86458ce2565a8f7b7bb1771881 move to 37bfedfd9766,40989,1732726116343 record at close sequenceid=5 2024-11-27T16:48:52,014 INFO [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:52,015 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e7d41a86458ce2565a8f7b7bb1771881, regionState=CLOSED 2024-11-27T16:48:52,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7d41a86458ce2565a8f7b7bb1771881, server=37bfedfd9766,37003,1732726116450 because future has completed 2024-11-27T16:48:52,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-27T16:48:52,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure e7d41a86458ce2565a8f7b7bb1771881, server=37bfedfd9766,37003,1732726116450 in 234 msec 2024-11-27T16:48:52,026 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, REOPEN/MOVE; state=CLOSED, location=37bfedfd9766,40989,1732726116343; forceNewPlan=false, retain=false 2024-11-27T16:48:52,177 INFO [37bfedfd9766:35959 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-27T16:48:52,178 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e7d41a86458ce2565a8f7b7bb1771881, regionState=OPENING, regionLocation=37bfedfd9766,40989,1732726116343 2024-11-27T16:48:52,183 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, REOPEN/MOVE because future has completed 2024-11-27T16:48:52,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7d41a86458ce2565a8f7b7bb1771881, server=37bfedfd9766,40989,1732726116343}] 2024-11-27T16:48:52,352 INFO [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:52,352 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => e7d41a86458ce2565a8f7b7bb1771881, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:48:52,353 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:52,353 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:52,353 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:52,353 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:52,360 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:52,361 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7d41a86458ce2565a8f7b7bb1771881 columnFamilyName cf1 2024-11-27T16:48:52,361 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:52,372 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf1/7bb6af69c06d4acb8c822ea1fea2f012 2024-11-27T16:48:52,372 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(327): Store=e7d41a86458ce2565a8f7b7bb1771881/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:52,372 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:52,373 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7d41a86458ce2565a8f7b7bb1771881 columnFamilyName cf2 2024-11-27T16:48:52,373 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:52,374 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(327): Store=e7d41a86458ce2565a8f7b7bb1771881/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:52,374 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:52,376 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:52,377 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:52,380 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:52,380 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:52,385 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-27T16:48:52,387 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:52,389 INFO [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened e7d41a86458ce2565a8f7b7bb1771881; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67162857, jitterRate=8.045583963394165E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-27T16:48:52,389 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:52,389 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for e7d41a86458ce2565a8f7b7bb1771881: Running coprocessor pre-open hook at 1732726132354Writing region info on filesystem at 1732726132354Initializing all the Stores at 1732726132356 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726132356Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726132358 (+2 ms)Cleaning up temporary data from old regions at 1732726132380 (+22 ms)Running coprocessor post-open hooks at 1732726132389 (+9 ms)Region opened successfully at 1732726132389 2024-11-27T16:48:52,391 INFO [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., pid=9, masterSystemTime=1732726132344 2024-11-27T16:48:52,394 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:52,395 INFO [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:52,396 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e7d41a86458ce2565a8f7b7bb1771881, regionState=OPEN, openSeqNum=9, regionLocation=37bfedfd9766,40989,1732726116343 2024-11-27T16:48:52,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7d41a86458ce2565a8f7b7bb1771881, server=37bfedfd9766,40989,1732726116343 because future has completed 2024-11-27T16:48:52,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-27T16:48:52,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure e7d41a86458ce2565a8f7b7bb1771881, server=37bfedfd9766,40989,1732726116343 in 214 msec 2024-11-27T16:48:52,411 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, REOPEN/MOVE in 632 msec 2024-11-27T16:48:52,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T16:48:52,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T16:48:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37003 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:38552 deadline: 1732726192455, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=37bfedfd9766 port=40989 startCode=1732726116343. As of locationSeqNum=5. 2024-11-27T16:48:52,465 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=37bfedfd9766 port=40989 startCode=1732726116343. As of locationSeqNum=5. 2024-11-27T16:48:52,466 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=37bfedfd9766 port=40989 startCode=1732726116343. As of locationSeqNum=5. 2024-11-27T16:48:52,466 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,40989,1732726116343, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=37bfedfd9766 port=40989 startCode=1732726116343. As of locationSeqNum=5. 2024-11-27T16:48:52,585 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e7d41a86458ce2565a8f7b7bb1771881 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-27T16:48:52,616 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/.tmp/cf1/11be5c723d964b5d8d8a04234b14b5be is 29, key is r1/cf1:/1732726132574/DeleteFamily/seqid=0 2024-11-27T16:48:52,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741888_1066 (size=4906) 2024-11-27T16:48:52,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741888_1066 (size=4906) 2024-11-27T16:48:52,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741888_1066 (size=4906) 2024-11-27T16:48:53,042 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/.tmp/cf1/11be5c723d964b5d8d8a04234b14b5be 2024-11-27T16:48:53,049 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 11be5c723d964b5d8d8a04234b14b5be 2024-11-27T16:48:53,070 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/.tmp/cf2/237f0f9b68704dffbc8063244e2b126d is 29, key is r1/cf2:/1732726132574/DeleteFamily/seqid=0 2024-11-27T16:48:53,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741889_1067 (size=4906) 2024-11-27T16:48:53,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741889_1067 (size=4906) 2024-11-27T16:48:53,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741889_1067 (size=4906) 2024-11-27T16:48:53,082 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/.tmp/cf2/237f0f9b68704dffbc8063244e2b126d 2024-11-27T16:48:53,089 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 237f0f9b68704dffbc8063244e2b126d 2024-11-27T16:48:53,090 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/.tmp/cf1/11be5c723d964b5d8d8a04234b14b5be as hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf1/11be5c723d964b5d8d8a04234b14b5be 2024-11-27T16:48:53,102 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 11be5c723d964b5d8d8a04234b14b5be 2024-11-27T16:48:53,102 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf1/11be5c723d964b5d8d8a04234b14b5be, entries=1, sequenceid=12, filesize=4.8 K 2024-11-27T16:48:53,104 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/.tmp/cf2/237f0f9b68704dffbc8063244e2b126d as hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf2/237f0f9b68704dffbc8063244e2b126d 2024-11-27T16:48:53,111 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 237f0f9b68704dffbc8063244e2b126d 2024-11-27T16:48:53,111 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf2/237f0f9b68704dffbc8063244e2b126d, entries=1, sequenceid=12, filesize=4.8 K 2024-11-27T16:48:53,113 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for e7d41a86458ce2565a8f7b7bb1771881 in 527ms, sequenceid=12, compaction requested=false 2024-11-27T16:48:53,113 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e7d41a86458ce2565a8f7b7bb1771881: 2024-11-27T16:48:53,116 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T16:48:53,118 DEBUG [Time-limited test {}] regionserver.HStore(1541): e7d41a86458ce2565a8f7b7bb1771881/cf1 is initiating major compaction (all files) 2024-11-27T16:48:53,118 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T16:48:53,118 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:53,119 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of e7d41a86458ce2565a8f7b7bb1771881/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:53,120 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf1/7bb6af69c06d4acb8c822ea1fea2f012, hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf1/11be5c723d964b5d8d8a04234b14b5be] into tmpdir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/.tmp, totalSize=9.5 K 2024-11-27T16:48:53,121 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 7bb6af69c06d4acb8c822ea1fea2f012, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732726131759 2024-11-27T16:48:53,121 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 11be5c723d964b5d8d8a04234b14b5be, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-27T16:48:53,136 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): e7d41a86458ce2565a8f7b7bb1771881#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T16:48:53,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741890_1068 (size=4626) 2024-11-27T16:48:53,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741890_1068 (size=4626) 2024-11-27T16:48:53,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741890_1068 (size=4626) 2024-11-27T16:48:53,164 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/.tmp/cf1/4ba09ba2609e4dfbad1a42fa44fe4e8a as hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf1/4ba09ba2609e4dfbad1a42fa44fe4e8a 2024-11-27T16:48:53,187 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in e7d41a86458ce2565a8f7b7bb1771881/cf1 of e7d41a86458ce2565a8f7b7bb1771881 into 4ba09ba2609e4dfbad1a42fa44fe4e8a(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T16:48:53,187 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for e7d41a86458ce2565a8f7b7bb1771881: 2024-11-27T16:48:53,187 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-27T16:48:53,187 DEBUG [Time-limited test {}] regionserver.HStore(1541): e7d41a86458ce2565a8f7b7bb1771881/cf2 is initiating major compaction (all files) 2024-11-27T16:48:53,187 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T16:48:53,187 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T16:48:53,188 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of e7d41a86458ce2565a8f7b7bb1771881/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:53,188 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf2/237f0f9b68704dffbc8063244e2b126d] into tmpdir=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/.tmp, totalSize=4.8 K 2024-11-27T16:48:53,189 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 237f0f9b68704dffbc8063244e2b126d, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-27T16:48:53,196 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): e7d41a86458ce2565a8f7b7bb1771881#cf2#compaction#17 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T16:48:53,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741891_1069 (size=4592) 2024-11-27T16:48:53,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741891_1069 (size=4592) 2024-11-27T16:48:53,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741891_1069 (size=4592) 2024-11-27T16:48:53,221 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/.tmp/cf2/068e1298a2464f688337f1346e54762b as hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf2/068e1298a2464f688337f1346e54762b 2024-11-27T16:48:53,229 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in e7d41a86458ce2565a8f7b7bb1771881/cf2 of e7d41a86458ce2565a8f7b7bb1771881 into 068e1298a2464f688337f1346e54762b(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T16:48:53,229 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for e7d41a86458ce2565a8f7b7bb1771881: 2024-11-27T16:48:53,235 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35959 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=e7d41a86458ce2565a8f7b7bb1771881, source=37bfedfd9766,40989,1732726116343, destination=37bfedfd9766,37003,1732726116450, warming up region on 37bfedfd9766,37003,1732726116450 2024-11-27T16:48:53,235 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35959 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=e7d41a86458ce2565a8f7b7bb1771881, source=37bfedfd9766,40989,1732726116343, destination=37bfedfd9766,37003,1732726116450, running balancer 2024-11-27T16:48:53,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35959 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, REOPEN/MOVE 2024-11-27T16:48:53,237 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, REOPEN/MOVE 2024-11-27T16:48:53,239 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37003 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:53,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37003 {}] regionserver.HRegion(7855): Warmup {ENCODED => e7d41a86458ce2565a8f7b7bb1771881, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:48:53,239 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e7d41a86458ce2565a8f7b7bb1771881, regionState=CLOSING, regionLocation=37bfedfd9766,40989,1732726116343 2024-11-27T16:48:53,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37003 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:53,240 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:53,241 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7d41a86458ce2565a8f7b7bb1771881 columnFamilyName cf1 2024-11-27T16:48:53,241 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:53,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, REOPEN/MOVE because future has completed 2024-11-27T16:48:53,243 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T16:48:53,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7d41a86458ce2565a8f7b7bb1771881, server=37bfedfd9766,40989,1732726116343}] 2024-11-27T16:48:53,250 INFO [StoreFileOpener-e7d41a86458ce2565a8f7b7bb1771881-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 11be5c723d964b5d8d8a04234b14b5be 2024-11-27T16:48:53,250 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf1/11be5c723d964b5d8d8a04234b14b5be 2024-11-27T16:48:53,255 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf1/4ba09ba2609e4dfbad1a42fa44fe4e8a 2024-11-27T16:48:53,261 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf1/7bb6af69c06d4acb8c822ea1fea2f012 2024-11-27T16:48:53,261 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(327): Store=e7d41a86458ce2565a8f7b7bb1771881/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:53,262 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:53,263 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7d41a86458ce2565a8f7b7bb1771881 columnFamilyName cf2 2024-11-27T16:48:53,263 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:53,270 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf2/068e1298a2464f688337f1346e54762b 2024-11-27T16:48:53,274 INFO [StoreFileOpener-e7d41a86458ce2565a8f7b7bb1771881-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 237f0f9b68704dffbc8063244e2b126d 2024-11-27T16:48:53,274 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf2/237f0f9b68704dffbc8063244e2b126d 2024-11-27T16:48:53,275 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(327): Store=e7d41a86458ce2565a8f7b7bb1771881/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:53,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37003 {}] regionserver.HRegion(1722): Closing e7d41a86458ce2565a8f7b7bb1771881, disabling compactions & flushes 2024-11-27T16:48:53,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37003 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:53,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37003 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:53,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37003 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. after waiting 0 ms 2024-11-27T16:48:53,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37003 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:53,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37003 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:53,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37003 {}] regionserver.HRegion(1676): Region close journal for e7d41a86458ce2565a8f7b7bb1771881: Waiting for close lock at 1732726133275Disabling compacts and flushes for region at 1732726133275Disabling writes for close at 1732726133275Writing region close event to WAL at 1732726133278 (+3 ms)Closed at 1732726133278 2024-11-27T16:48:53,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35959 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-27T16:48:53,398 INFO [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:53,398 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T16:48:53,398 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing e7d41a86458ce2565a8f7b7bb1771881, disabling compactions & flushes 2024-11-27T16:48:53,398 INFO [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:53,399 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:53,399 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. after waiting 0 ms 2024-11-27T16:48:53,399 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:53,399 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf1/7bb6af69c06d4acb8c822ea1fea2f012, hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf1/11be5c723d964b5d8d8a04234b14b5be] to archive 2024-11-27T16:48:53,403 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T16:48:53,406 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf1/7bb6af69c06d4acb8c822ea1fea2f012 to hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf1/7bb6af69c06d4acb8c822ea1fea2f012 2024-11-27T16:48:53,408 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf1/11be5c723d964b5d8d8a04234b14b5be to hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf1/11be5c723d964b5d8d8a04234b14b5be 2024-11-27T16:48:53,423 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf2/237f0f9b68704dffbc8063244e2b126d] to archive 2024-11-27T16:48:53,425 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T16:48:53,427 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf2/237f0f9b68704dffbc8063244e2b126d to hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf2/237f0f9b68704dffbc8063244e2b126d 2024-11-27T16:48:53,433 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-27T16:48:53,433 INFO [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:53,434 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for e7d41a86458ce2565a8f7b7bb1771881: Waiting for close lock at 1732726133398Running coprocessor pre-close hooks at 1732726133398Disabling compacts and flushes for region at 1732726133398Disabling writes for close at 1732726133399 (+1 ms)Writing region close event to WAL at 1732726133429 (+30 ms)Running coprocessor post-close hooks at 1732726133433 (+4 ms)Closed at 1732726133433 2024-11-27T16:48:53,434 INFO [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding e7d41a86458ce2565a8f7b7bb1771881 move to 37bfedfd9766,37003,1732726116450 record at close sequenceid=12 2024-11-27T16:48:53,437 INFO [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:53,438 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e7d41a86458ce2565a8f7b7bb1771881, regionState=CLOSED 2024-11-27T16:48:53,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7d41a86458ce2565a8f7b7bb1771881, server=37bfedfd9766,40989,1732726116343 because future has completed 2024-11-27T16:48:53,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-27T16:48:53,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure e7d41a86458ce2565a8f7b7bb1771881, server=37bfedfd9766,40989,1732726116343 in 204 msec 2024-11-27T16:48:53,452 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, REOPEN/MOVE; state=CLOSED, location=37bfedfd9766,37003,1732726116450; forceNewPlan=false, retain=false 2024-11-27T16:48:53,603 INFO [37bfedfd9766:35959 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-27T16:48:53,603 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e7d41a86458ce2565a8f7b7bb1771881, regionState=OPENING, regionLocation=37bfedfd9766,37003,1732726116450 2024-11-27T16:48:53,607 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, REOPEN/MOVE because future has completed 2024-11-27T16:48:53,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7d41a86458ce2565a8f7b7bb1771881, server=37bfedfd9766,37003,1732726116450}] 2024-11-27T16:48:53,765 INFO [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:53,765 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => e7d41a86458ce2565a8f7b7bb1771881, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:48:53,766 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:53,766 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:53,766 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:53,766 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:53,768 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:53,769 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7d41a86458ce2565a8f7b7bb1771881 columnFamilyName cf1 2024-11-27T16:48:53,769 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:53,778 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf1/4ba09ba2609e4dfbad1a42fa44fe4e8a 2024-11-27T16:48:53,778 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(327): Store=e7d41a86458ce2565a8f7b7bb1771881/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:53,778 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:53,779 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7d41a86458ce2565a8f7b7bb1771881 columnFamilyName cf2 2024-11-27T16:48:53,780 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:53,787 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf2/068e1298a2464f688337f1346e54762b 2024-11-27T16:48:53,787 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(327): Store=e7d41a86458ce2565a8f7b7bb1771881/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:53,787 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:53,788 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:53,790 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:53,791 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:53,791 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:53,792 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-27T16:48:53,793 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:53,794 INFO [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened e7d41a86458ce2565a8f7b7bb1771881; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67186417, jitterRate=0.0011556297540664673}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-27T16:48:53,794 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:53,795 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for e7d41a86458ce2565a8f7b7bb1771881: Running coprocessor pre-open hook at 1732726133766Writing region info on filesystem at 1732726133766Initializing all the Stores at 1732726133767 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726133767Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726133768 (+1 ms)Cleaning up temporary data from old regions at 1732726133791 (+23 ms)Running coprocessor post-open hooks at 1732726133794 (+3 ms)Region opened successfully at 1732726133795 (+1 ms) 2024-11-27T16:48:53,797 INFO [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., pid=12, masterSystemTime=1732726133760 2024-11-27T16:48:53,801 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e7d41a86458ce2565a8f7b7bb1771881, regionState=OPEN, openSeqNum=18, regionLocation=37bfedfd9766,37003,1732726116450 2024-11-27T16:48:53,804 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:53,804 INFO [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:53,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7d41a86458ce2565a8f7b7bb1771881, server=37bfedfd9766,37003,1732726116450 because future has completed 2024-11-27T16:48:53,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-27T16:48:53,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure e7d41a86458ce2565a8f7b7bb1771881, server=37bfedfd9766,37003,1732726116450 in 199 msec 2024-11-27T16:48:53,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, REOPEN/MOVE in 579 msec 2024-11-27T16:48:53,839 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T16:48:53,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T16:48:53,844 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 37bfedfd9766,37003,1732726116450: testing ***** 2024-11-27T16:48:53,844 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-27T16:48:53,846 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-27T16:48:53,848 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-27T16:48:53,851 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-27T16:48:53,852 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-27T16:48:53,863 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 406902712 }, "NonHeapMemoryUsage": { "committed": 172425216, "init": 7667712, "max": -1, "used": 169786032 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "37bfedfd9766", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2068, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 12, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 6, "ProcessCallTime_75th_percentile": 9, "ProcessCallTime_90th_percentile": 10, "ProcessCallTime_95th_percentile": 11, "ProcessCallTime_98th_percentile": 11, "ProcessCallTime_99th_percentile": 11, "ProcessCallTime_99.9th_percentile": 11, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 12, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 6, "TotalCallTime_75th_percentile": 9, "TotalCallTime_90th_percentile": 10, "TotalCallTime_95th_percentile": 11, "TotalCallTime_98th_percentile": 11, "TotalCallTime_99th_percentile": 11, "TotalCallTime_99.9th_percentile": 11, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "37bfedfd9766", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:60075", "tag.serverName": "37bfedfd9766,36103,1732726116491", "tag.clusterId": "7a7dfc1d-b607-4ef1-a912-89ad055552df", "tag.Context": "regionserver", "tag.Hostname": "37bfedfd9766", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732726116491, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-27T16:48:53,867 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35959 {}] master.MasterRpcServices(700): 37bfedfd9766,37003,1732726116450 reported a fatal error: ***** ABORTING region server 37bfedfd9766,37003,1732726116450: testing ***** 2024-11-27T16:48:53,869 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '37bfedfd9766,37003,1732726116450' ***** 2024-11-27T16:48:53,869 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-27T16:48:53,870 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T16:48:53,871 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T16:48:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:57574 deadline: 1732726193870, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=37bfedfd9766 port=37003 startCode=1732726116450. As of locationSeqNum=12. 2024-11-27T16:48:53,871 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,40989,1732726116343, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,40989,1732726116343, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=37bfedfd9766 port=37003 startCode=1732726116450. As of locationSeqNum=12. 2024-11-27T16:48:53,872 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,40989,1732726116343, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=37bfedfd9766 port=37003 startCode=1732726116450. As of locationSeqNum=12. 2024-11-27T16:48:53,872 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,40989,1732726116343, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=37bfedfd9766 port=37003 startCode=1732726116450. As of locationSeqNum=12. 2024-11-27T16:48:53,872 INFO [RS:1;37bfedfd9766:37003 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-27T16:48:53,872 INFO [RS:1;37bfedfd9766:37003 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-27T16:48:53,872 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.HRegionServer(3091): Received CLOSE for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:53,873 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.HRegionServer(956): aborting server 37bfedfd9766,37003,1732726116450 2024-11-27T16:48:53,873 INFO [RS:1;37bfedfd9766:37003 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T16:48:53,873 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e7d41a86458ce2565a8f7b7bb1771881, disabling compactions & flushes 2024-11-27T16:48:53,873 INFO [RS:1;37bfedfd9766:37003 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;37bfedfd9766:37003. 2024-11-27T16:48:53,873 INFO [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:53,873 DEBUG [RS:1;37bfedfd9766:37003 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T16:48:53,873 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:53,874 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. after waiting 0 ms 2024-11-27T16:48:53,874 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:53,874 DEBUG [RS:1;37bfedfd9766:37003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T16:48:53,876 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-27T16:48:53,876 DEBUG [RS:1;37bfedfd9766:37003 {}] regionserver.HRegionServer(1325): Online Regions={e7d41a86458ce2565a8f7b7bb1771881=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.} 2024-11-27T16:48:53,876 DEBUG [RS:1;37bfedfd9766:37003 {}] regionserver.HRegionServer(1351): Waiting on e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:53,877 INFO [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:53,877 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e7d41a86458ce2565a8f7b7bb1771881: Waiting for close lock at 1732726133873Running coprocessor pre-close hooks at 1732726133873Disabling compacts and flushes for region at 1732726133873Disabling writes for close at 1732726133874 (+1 ms)Writing region close event to WAL at 1732726133876 (+2 ms)Running coprocessor post-close hooks at 1732726133876Closed at 1732726133877 (+1 ms) 2024-11-27T16:48:53,877 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:53,931 INFO [regionserver/37bfedfd9766:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T16:48:53,983 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 37bfedfd9766,37003,1732726116450 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:48:53,984 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 37bfedfd9766,37003,1732726116450 aborting 2024-11-27T16:48:53,984 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 37bfedfd9766,37003,1732726116450 aborting 2024-11-27T16:48:53,984 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=12 from cache 2024-11-27T16:48:54,077 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.HRegionServer(976): stopping server 37bfedfd9766,37003,1732726116450; all regions closed. 2024-11-27T16:48:54,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741834_1010 (size=1405) 2024-11-27T16:48:54,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741834_1010 (size=1405) 2024-11-27T16:48:54,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741834_1010 (size=1405) 2024-11-27T16:48:54,082 DEBUG [RS:1;37bfedfd9766:37003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T16:48:54,082 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T16:48:54,082 INFO [RS:1;37bfedfd9766:37003 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T16:48:54,083 INFO [RS:1;37bfedfd9766:37003 {}] hbase.ChoreService(370): Chore service for: regionserver/37bfedfd9766:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T16:48:54,083 INFO [regionserver/37bfedfd9766:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T16:48:54,083 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T16:48:54,083 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T16:48:54,083 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T16:48:54,083 INFO [RS:1;37bfedfd9766:37003 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T16:48:54,084 INFO [RS:1;37bfedfd9766:37003 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37003 2024-11-27T16:48:54,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37003-0x10039e202d30002, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/37bfedfd9766,37003,1732726116450 2024-11-27T16:48:54,093 INFO [RS:1;37bfedfd9766:37003 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T16:48:54,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T16:48:54,096 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [37bfedfd9766,37003,1732726116450] 2024-11-27T16:48:54,098 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/37bfedfd9766,37003,1732726116450 already deleted, retry=false 2024-11-27T16:48:54,098 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 37bfedfd9766,37003,1732726116450 on 37bfedfd9766,35959,1732726115618 2024-11-27T16:48:54,104 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 37bfedfd9766,37003,1732726116450, splitWal=true, meta=false 2024-11-27T16:48:54,107 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 37bfedfd9766,37003,1732726116450 (carryingMeta=false) 37bfedfd9766,37003,1732726116450/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@3535cc71[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-27T16:48:54,107 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 37bfedfd9766,37003,1732726116450, splitWal=true, meta=false 2024-11-27T16:48:54,109 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 37bfedfd9766,37003,1732726116450 had 1 regions 2024-11-27T16:48:54,110 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 37bfedfd9766,37003,1732726116450, splitWal=true, meta=false, isMeta: false 2024-11-27T16:48:54,112 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450-splitting 2024-11-27T16:48:54,113 INFO [PEWorker-1 {}] master.SplitWALManager(105): 37bfedfd9766,37003,1732726116450 WAL count=1, meta=false 2024-11-27T16:48:54,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 37bfedfd9766%2C37003%2C1732726116450.1732726117955}] 2024-11-27T16:48:54,123 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=37bfedfd9766,36103,1732726116491 2024-11-27T16:48:54,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 37bfedfd9766%2C37003%2C1732726116450.1732726117955, worker=37bfedfd9766,36103,1732726116491}] 2024-11-27T16:48:54,196 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=18] 2024-11-27T16:48:54,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37003-0x10039e202d30002, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T16:48:54,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37003-0x10039e202d30002, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T16:48:54,197 INFO [RS:1;37bfedfd9766:37003 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T16:48:54,197 INFO [RS:1;37bfedfd9766:37003 {}] regionserver.HRegionServer(1031): Exiting; stopping=37bfedfd9766,37003,1732726116450; zookeeper connection closed. 2024-11-27T16:48:54,197 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@649b4f11 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@649b4f11 2024-11-27T16:48:54,198 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 37bfedfd9766:37003 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 37bfedfd9766/172.17.0.2:37003 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:48:54,199 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=18, error=java.net.ConnectException: Call to address=37bfedfd9766:37003 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 37bfedfd9766/172.17.0.2:37003 2024-11-27T16:48:54,199 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-27T16:48:54,199 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=18 from cache 2024-11-27T16:48:54,199 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 37bfedfd9766:37003 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 37bfedfd9766/172.17.0.2:37003 2024-11-27T16:48:54,286 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T16:48:54,287 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49763, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T16:48:54,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36103 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-27T16:48:54,307 INFO [RS_LOG_REPLAY_OPS-regionserver/37bfedfd9766:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450-splitting/37bfedfd9766%2C37003%2C1732726116450.1732726117955, size=1.4 K (1405bytes) 2024-11-27T16:48:54,307 INFO [RS_LOG_REPLAY_OPS-regionserver/37bfedfd9766:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450-splitting/37bfedfd9766%2C37003%2C1732726116450.1732726117955 2024-11-27T16:48:54,307 INFO [RS_LOG_REPLAY_OPS-regionserver/37bfedfd9766:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450-splitting/37bfedfd9766%2C37003%2C1732726116450.1732726117955 after 0ms 2024-11-27T16:48:54,310 DEBUG [RS_LOG_REPLAY_OPS-regionserver/37bfedfd9766:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450-splitting/37bfedfd9766%2C37003%2C1732726116450.1732726117955: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:54,310 INFO [RS_LOG_REPLAY_OPS-regionserver/37bfedfd9766:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450-splitting/37bfedfd9766%2C37003%2C1732726116450.1732726117955 took 4ms 2024-11-27T16:48:54,317 DEBUG [RS_LOG_REPLAY_OPS-regionserver/37bfedfd9766:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for e7d41a86458ce2565a8f7b7bb1771881: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-27T16:48:54,318 DEBUG [RS_LOG_REPLAY_OPS-regionserver/37bfedfd9766:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450-splitting/37bfedfd9766%2C37003%2C1732726116450.1732726117955 so closing down 2024-11-27T16:48:54,318 DEBUG [RS_LOG_REPLAY_OPS-regionserver/37bfedfd9766:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T16:48:54,318 INFO [RS_LOG_REPLAY_OPS-regionserver/37bfedfd9766:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T16:48:54,318 INFO [RS_LOG_REPLAY_OPS-regionserver/37bfedfd9766:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450-splitting/37bfedfd9766%2C37003%2C1732726116450.1732726117955, size=1.4 K, length=1405, corrupted=false, cancelled=false 2024-11-27T16:48:54,318 DEBUG [RS_LOG_REPLAY_OPS-regionserver/37bfedfd9766:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450-splitting/37bfedfd9766%2C37003%2C1732726116450.1732726117955, journal: Splitting hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450-splitting/37bfedfd9766%2C37003%2C1732726116450.1732726117955, size=1.4 K (1405bytes) at 1732726134307Finishing writing output for hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450-splitting/37bfedfd9766%2C37003%2C1732726116450.1732726117955 so closing down at 1732726134318 (+11 ms)3 split writer threads finished at 1732726134318Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450-splitting/37bfedfd9766%2C37003%2C1732726116450.1732726117955, size=1.4 K, length=1405, corrupted=false, cancelled=false at 1732726134318 2024-11-27T16:48:54,318 DEBUG [RS_LOG_REPLAY_OPS-regionserver/37bfedfd9766:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450-splitting/37bfedfd9766%2C37003%2C1732726116450.1732726117955 2024-11-27T16:48:54,319 DEBUG [RS_LOG_REPLAY_OPS-regionserver/37bfedfd9766:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-27T16:48:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35959 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-27T16:48:54,326 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450-splitting/37bfedfd9766%2C37003%2C1732726116450.1732726117955 to hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/oldWALs 2024-11-27T16:48:54,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-27T16:48:54,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 37bfedfd9766%2C37003%2C1732726116450.1732726117955, worker=37bfedfd9766,36103,1732726116491 in 201 msec 2024-11-27T16:48:54,331 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=37bfedfd9766,36103,1732726116491 2024-11-27T16:48:54,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-27T16:48:54,334 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 37bfedfd9766%2C37003%2C1732726116450.1732726117955, worker=37bfedfd9766,36103,1732726116491 in 216 msec 2024-11-27T16:48:54,335 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450-splitting dir is empty, no logs to split. 2024-11-27T16:48:54,335 INFO [PEWorker-1 {}] master.SplitWALManager(105): 37bfedfd9766,37003,1732726116450 WAL count=0, meta=false 2024-11-27T16:48:54,335 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 37bfedfd9766,37003,1732726116450 WAL splitting is done? wals=0, meta=false 2024-11-27T16:48:54,338 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 37bfedfd9766,37003,1732726116450 failed, ignore...File hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/WALs/37bfedfd9766,37003,1732726116450-splitting does not exist. 2024-11-27T16:48:54,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, ASSIGN}] 2024-11-27T16:48:54,341 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, ASSIGN 2024-11-27T16:48:54,342 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T16:48:54,493 DEBUG [37bfedfd9766:35959 {}] balancer.BalancerClusterState(204): Hosts are {37bfedfd9766=0} racks are {/default-rack=0} 2024-11-27T16:48:54,493 DEBUG [37bfedfd9766:35959 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T16:48:54,493 DEBUG [37bfedfd9766:35959 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T16:48:54,493 DEBUG [37bfedfd9766:35959 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T16:48:54,493 DEBUG [37bfedfd9766:35959 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T16:48:54,493 INFO [37bfedfd9766:35959 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T16:48:54,493 INFO [37bfedfd9766:35959 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T16:48:54,493 DEBUG [37bfedfd9766:35959 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T16:48:54,493 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=e7d41a86458ce2565a8f7b7bb1771881, regionState=OPENING, regionLocation=37bfedfd9766,40989,1732726116343 2024-11-27T16:48:54,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, ASSIGN because future has completed 2024-11-27T16:48:54,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7d41a86458ce2565a8f7b7bb1771881, server=37bfedfd9766,40989,1732726116343}] 2024-11-27T16:48:54,504 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=18] 2024-11-27T16:48:54,505 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 37bfedfd9766:37003 this server is in the failed servers list 2024-11-27T16:48:54,505 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=37bfedfd9766:37003 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 37bfedfd9766:37003 2024-11-27T16:48:54,505 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 37bfedfd9766:37003 2024-11-27T16:48:54,506 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,37003,1732726116450, seqNum=18 from cache 2024-11-27T16:48:54,654 INFO [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:54,655 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => e7d41a86458ce2565a8f7b7bb1771881, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:48:54,655 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:54,655 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:54,655 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:54,655 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:54,657 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:54,658 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7d41a86458ce2565a8f7b7bb1771881 columnFamilyName cf1 2024-11-27T16:48:54,658 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:54,665 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf1/4ba09ba2609e4dfbad1a42fa44fe4e8a 2024-11-27T16:48:54,665 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(327): Store=e7d41a86458ce2565a8f7b7bb1771881/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:54,665 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:54,666 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7d41a86458ce2565a8f7b7bb1771881 columnFamilyName cf2 2024-11-27T16:48:54,666 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:54,672 DEBUG [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/cf2/068e1298a2464f688337f1346e54762b 2024-11-27T16:48:54,672 INFO [StoreOpener-e7d41a86458ce2565a8f7b7bb1771881-1 {}] regionserver.HStore(327): Store=e7d41a86458ce2565a8f7b7bb1771881/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:54,673 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:54,674 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:54,675 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:54,676 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:54,676 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:54,677 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-27T16:48:54,679 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:54,680 INFO [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened e7d41a86458ce2565a8f7b7bb1771881; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71268627, jitterRate=0.06198529899120331}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-27T16:48:54,680 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:48:54,681 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for e7d41a86458ce2565a8f7b7bb1771881: Running coprocessor pre-open hook at 1732726134655Writing region info on filesystem at 1732726134655Initializing all the Stores at 1732726134656 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726134656Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726134657 (+1 ms)Cleaning up temporary data from old regions at 1732726134676 (+19 ms)Running coprocessor post-open hooks at 1732726134681 (+5 ms)Region opened successfully at 1732726134681 2024-11-27T16:48:54,682 INFO [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., pid=17, masterSystemTime=1732726134650 2024-11-27T16:48:54,686 DEBUG [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:54,686 INFO [RS_OPEN_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:48:54,687 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=e7d41a86458ce2565a8f7b7bb1771881, regionState=OPEN, openSeqNum=18, regionLocation=37bfedfd9766,40989,1732726116343 2024-11-27T16:48:54,688 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35959 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=37bfedfd9766,40989,1732726116343, table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T16:48:54,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7d41a86458ce2565a8f7b7bb1771881, server=37bfedfd9766,40989,1732726116343 because future has completed 2024-11-27T16:48:54,694 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-27T16:48:54,694 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure e7d41a86458ce2565a8f7b7bb1771881, server=37bfedfd9766,40989,1732726116343 in 194 msec 2024-11-27T16:48:54,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-27T16:48:54,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e7d41a86458ce2565a8f7b7bb1771881, ASSIGN in 354 msec 2024-11-27T16:48:54,700 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 37bfedfd9766,37003,1732726116450 after splitting done 2024-11-27T16:48:54,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 37bfedfd9766,37003,1732726116450, splitWal=true, meta=false in 601 msec 2024-11-27T16:48:55,015 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881., hostname=37bfedfd9766,40989,1732726116343, seqNum=18] 2024-11-27T16:48:55,031 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=408 (was 412), OpenFileDescriptor=1048 (was 1022) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=273 (was 254) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6430 (was 6600) 2024-11-27T16:48:55,033 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-27T16:48:55,046 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=408, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=273, ProcessCount=11, AvailableMemoryMB=6430 2024-11-27T16:48:55,046 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-27T16:48:55,062 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:55,063 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:55,064 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T16:48:55,067 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-54236919, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/hregion-54236919, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:55,080 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-54236919/hregion-54236919.1732726135067, exclude list is [], retry=0 2024-11-27T16:48:55,082 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:55,083 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:55,083 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:55,087 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-54236919/hregion-54236919.1732726135067 2024-11-27T16:48:55,088 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:39655:39655)] 2024-11-27T16:48:55,088 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4630e7dfe17ac910f657c7ced8cc5f5d, NAME => 'testReplayEditsWrittenViaHRegion,,1732726135062.4630e7dfe17ac910f657c7ced8cc5f5d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40409/hbase 2024-11-27T16:48:55,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741893_1071 (size=67) 2024-11-27T16:48:55,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741893_1071 (size=67) 2024-11-27T16:48:55,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741893_1071 (size=67) 2024-11-27T16:48:55,097 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732726135062.4630e7dfe17ac910f657c7ced8cc5f5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:55,099 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,100 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4630e7dfe17ac910f657c7ced8cc5f5d columnFamilyName a 2024-11-27T16:48:55,100 DEBUG [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:55,101 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.HStore(327): Store=4630e7dfe17ac910f657c7ced8cc5f5d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:55,101 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,102 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4630e7dfe17ac910f657c7ced8cc5f5d columnFamilyName b 2024-11-27T16:48:55,103 DEBUG [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:55,103 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.HStore(327): Store=4630e7dfe17ac910f657c7ced8cc5f5d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:55,103 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,105 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4630e7dfe17ac910f657c7ced8cc5f5d columnFamilyName c 2024-11-27T16:48:55,105 DEBUG [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:55,105 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.HStore(327): Store=4630e7dfe17ac910f657c7ced8cc5f5d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:55,105 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,106 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,106 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,107 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,107 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,108 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T16:48:55,109 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,111 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T16:48:55,112 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4630e7dfe17ac910f657c7ced8cc5f5d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68464246, jitterRate=0.020196765661239624}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T16:48:55,112 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4630e7dfe17ac910f657c7ced8cc5f5d: Writing region info on filesystem at 1732726135097Initializing all the Stores at 1732726135098 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726135098Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726135099 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726135099Cleaning up temporary data from old regions at 1732726135107 (+8 ms)Region opened successfully at 1732726135112 (+5 ms) 2024-11-27T16:48:55,112 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4630e7dfe17ac910f657c7ced8cc5f5d, disabling compactions & flushes 2024-11-27T16:48:55,112 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732726135062.4630e7dfe17ac910f657c7ced8cc5f5d. 2024-11-27T16:48:55,112 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732726135062.4630e7dfe17ac910f657c7ced8cc5f5d. 2024-11-27T16:48:55,112 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732726135062.4630e7dfe17ac910f657c7ced8cc5f5d. after waiting 0 ms 2024-11-27T16:48:55,112 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732726135062.4630e7dfe17ac910f657c7ced8cc5f5d. 2024-11-27T16:48:55,117 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732726135062.4630e7dfe17ac910f657c7ced8cc5f5d. 2024-11-27T16:48:55,117 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4630e7dfe17ac910f657c7ced8cc5f5d: Waiting for close lock at 1732726135112Disabling compacts and flushes for region at 1732726135112Disabling writes for close at 1732726135112Writing region close event to WAL at 1732726135117 (+5 ms)Closed at 1732726135117 2024-11-27T16:48:55,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741892_1070 (size=95) 2024-11-27T16:48:55,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741892_1070 (size=95) 2024-11-27T16:48:55,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741892_1070 (size=95) 2024-11-27T16:48:55,122 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T16:48:55,122 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-54236919:(num 1732726135067) 2024-11-27T16:48:55,122 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:48:55,123 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:55,137 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061/wal.1732726135123, exclude list is [], retry=0 2024-11-27T16:48:55,139 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:55,140 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:55,140 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:55,142 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061/wal.1732726135123 2024-11-27T16:48:55,142 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:42883:42883)] 2024-11-27T16:48:55,142 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4630e7dfe17ac910f657c7ced8cc5f5d, NAME => 'testReplayEditsWrittenViaHRegion,,1732726135062.4630e7dfe17ac910f657c7ced8cc5f5d.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:48:55,142 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732726135062.4630e7dfe17ac910f657c7ced8cc5f5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:55,143 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,143 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,144 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,145 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4630e7dfe17ac910f657c7ced8cc5f5d columnFamilyName a 2024-11-27T16:48:55,145 DEBUG [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:55,145 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.HStore(327): Store=4630e7dfe17ac910f657c7ced8cc5f5d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:55,145 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,146 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4630e7dfe17ac910f657c7ced8cc5f5d columnFamilyName b 2024-11-27T16:48:55,146 DEBUG [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:55,147 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.HStore(327): Store=4630e7dfe17ac910f657c7ced8cc5f5d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:55,147 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,147 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4630e7dfe17ac910f657c7ced8cc5f5d columnFamilyName c 2024-11-27T16:48:55,147 DEBUG [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:55,148 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.HStore(327): Store=4630e7dfe17ac910f657c7ced8cc5f5d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:55,148 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,149 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,150 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,151 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,151 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,152 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T16:48:55,154 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,155 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4630e7dfe17ac910f657c7ced8cc5f5d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75495120, jitterRate=0.12496495246887207}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T16:48:55,155 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4630e7dfe17ac910f657c7ced8cc5f5d: Writing region info on filesystem at 1732726135143Initializing all the Stores at 1732726135143Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726135143Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726135144 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726135144Cleaning up temporary data from old regions at 1732726135152 (+8 ms)Region opened successfully at 1732726135155 (+3 ms) 2024-11-27T16:48:55,190 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4630e7dfe17ac910f657c7ced8cc5f5d 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-27T16:48:55,211 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/.tmp/a/ec92b23c284f462192d9701b5beb43cf is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732726135155/Put/seqid=0 2024-11-27T16:48:55,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741895_1073 (size=5958) 2024-11-27T16:48:55,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741895_1073 (size=5958) 2024-11-27T16:48:55,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741895_1073 (size=5958) 2024-11-27T16:48:55,218 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/.tmp/a/ec92b23c284f462192d9701b5beb43cf 2024-11-27T16:48:55,239 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/.tmp/b/176cdad452624ef1beae5a4e467f9dfb is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732726135165/Put/seqid=0 2024-11-27T16:48:55,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741896_1074 (size=5958) 2024-11-27T16:48:55,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741896_1074 (size=5958) 2024-11-27T16:48:55,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741896_1074 (size=5958) 2024-11-27T16:48:55,247 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/.tmp/b/176cdad452624ef1beae5a4e467f9dfb 2024-11-27T16:48:55,267 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/.tmp/c/35dbd015ed12451e8779626833bffbb4 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732726135176/Put/seqid=0 2024-11-27T16:48:55,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741897_1075 (size=5958) 2024-11-27T16:48:55,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741897_1075 (size=5958) 2024-11-27T16:48:55,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741897_1075 (size=5958) 2024-11-27T16:48:55,275 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/.tmp/c/35dbd015ed12451e8779626833bffbb4 2024-11-27T16:48:55,282 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/.tmp/a/ec92b23c284f462192d9701b5beb43cf as hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/a/ec92b23c284f462192d9701b5beb43cf 2024-11-27T16:48:55,288 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/a/ec92b23c284f462192d9701b5beb43cf, entries=10, sequenceid=33, filesize=5.8 K 2024-11-27T16:48:55,289 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/.tmp/b/176cdad452624ef1beae5a4e467f9dfb as hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/b/176cdad452624ef1beae5a4e467f9dfb 2024-11-27T16:48:55,294 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/b/176cdad452624ef1beae5a4e467f9dfb, entries=10, sequenceid=33, filesize=5.8 K 2024-11-27T16:48:55,295 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/.tmp/c/35dbd015ed12451e8779626833bffbb4 as hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/c/35dbd015ed12451e8779626833bffbb4 2024-11-27T16:48:55,300 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/c/35dbd015ed12451e8779626833bffbb4, entries=10, sequenceid=33, filesize=5.8 K 2024-11-27T16:48:55,302 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 4630e7dfe17ac910f657c7ced8cc5f5d in 112ms, sequenceid=33, compaction requested=false 2024-11-27T16:48:55,302 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4630e7dfe17ac910f657c7ced8cc5f5d: 2024-11-27T16:48:55,302 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4630e7dfe17ac910f657c7ced8cc5f5d, disabling compactions & flushes 2024-11-27T16:48:55,302 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732726135062.4630e7dfe17ac910f657c7ced8cc5f5d. 2024-11-27T16:48:55,302 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732726135062.4630e7dfe17ac910f657c7ced8cc5f5d. 2024-11-27T16:48:55,302 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732726135062.4630e7dfe17ac910f657c7ced8cc5f5d. after waiting 0 ms 2024-11-27T16:48:55,302 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732726135062.4630e7dfe17ac910f657c7ced8cc5f5d. 2024-11-27T16:48:55,303 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732726135062.4630e7dfe17ac910f657c7ced8cc5f5d. 2024-11-27T16:48:55,303 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4630e7dfe17ac910f657c7ced8cc5f5d: Waiting for close lock at 1732726135302Disabling compacts and flushes for region at 1732726135302Disabling writes for close at 1732726135302Writing region close event to WAL at 1732726135303 (+1 ms)Closed at 1732726135303 2024-11-27T16:48:55,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741894_1072 (size=3384) 2024-11-27T16:48:55,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741894_1072 (size=3384) 2024-11-27T16:48:55,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741894_1072 (size=3384) 2024-11-27T16:48:55,312 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/b/176cdad452624ef1beae5a4e467f9dfb to hdfs://localhost:40409/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/b/176cdad452624ef1beae5a4e467f9dfb 2024-11-27T16:48:55,328 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40409/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061/wal.1732726135123, size=3.3 K (3384bytes) 2024-11-27T16:48:55,328 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40409/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061/wal.1732726135123 2024-11-27T16:48:55,329 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40409/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061/wal.1732726135123 after 1ms 2024-11-27T16:48:55,331 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061/wal.1732726135123: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:55,331 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40409/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061/wal.1732726135123 took 3ms 2024-11-27T16:48:55,335 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40409/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061/wal.1732726135123 so closing down 2024-11-27T16:48:55,335 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T16:48:55,337 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732726135123.temp 2024-11-27T16:48:55,338 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/recovered.edits/0000000000000000003-wal.1732726135123.temp 2024-11-27T16:48:55,339 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T16:48:55,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741898_1076 (size=2944) 2024-11-27T16:48:55,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741898_1076 (size=2944) 2024-11-27T16:48:55,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741898_1076 (size=2944) 2024-11-27T16:48:55,351 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/recovered.edits/0000000000000000003-wal.1732726135123.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-27T16:48:55,353 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/recovered.edits/0000000000000000003-wal.1732726135123.temp to hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/recovered.edits/0000000000000000032 2024-11-27T16:48:55,353 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 21 ms; skipped=2; WAL=hdfs://localhost:40409/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061/wal.1732726135123, size=3.3 K, length=3384, corrupted=false, cancelled=false 2024-11-27T16:48:55,353 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40409/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061/wal.1732726135123, journal: Splitting hdfs://localhost:40409/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061/wal.1732726135123, size=3.3 K (3384bytes) at 1732726135328Finishing writing output for hdfs://localhost:40409/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061/wal.1732726135123 so closing down at 1732726135335 (+7 ms)Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/recovered.edits/0000000000000000003-wal.1732726135123.temp at 1732726135338 (+3 ms)3 split writer threads finished at 1732726135339 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/recovered.edits/0000000000000000003-wal.1732726135123.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732726135351 (+12 ms)Rename recovered edits hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/recovered.edits/0000000000000000003-wal.1732726135123.temp to hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/recovered.edits/0000000000000000032 at 1732726135353 (+2 ms)Processed 32 edits across 1 Regions in 21 ms; skipped=2; WAL=hdfs://localhost:40409/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061/wal.1732726135123, size=3.3 K, length=3384, corrupted=false, cancelled=false at 1732726135353 2024-11-27T16:48:55,355 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40409/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061/wal.1732726135123 to hdfs://localhost:40409/hbase/oldWALs/wal.1732726135123 2024-11-27T16:48:55,356 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/recovered.edits/0000000000000000032 2024-11-27T16:48:55,356 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:48:55,358 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:55,372 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061/wal.1732726135359, exclude list is [], retry=0 2024-11-27T16:48:55,375 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:55,375 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:55,376 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:55,377 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061/wal.1732726135359 2024-11-27T16:48:55,377 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:42883:42883)] 2024-11-27T16:48:55,377 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4630e7dfe17ac910f657c7ced8cc5f5d, NAME => 'testReplayEditsWrittenViaHRegion,,1732726135062.4630e7dfe17ac910f657c7ced8cc5f5d.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:48:55,378 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732726135062.4630e7dfe17ac910f657c7ced8cc5f5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:55,378 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,378 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,379 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,381 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4630e7dfe17ac910f657c7ced8cc5f5d columnFamilyName a 2024-11-27T16:48:55,381 DEBUG [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:55,387 DEBUG [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/a/ec92b23c284f462192d9701b5beb43cf 2024-11-27T16:48:55,387 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.HStore(327): Store=4630e7dfe17ac910f657c7ced8cc5f5d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:55,387 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,389 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4630e7dfe17ac910f657c7ced8cc5f5d columnFamilyName b 2024-11-27T16:48:55,389 DEBUG [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:55,389 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.HStore(327): Store=4630e7dfe17ac910f657c7ced8cc5f5d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:55,389 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,390 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4630e7dfe17ac910f657c7ced8cc5f5d columnFamilyName c 2024-11-27T16:48:55,390 DEBUG [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:55,395 DEBUG [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/c/35dbd015ed12451e8779626833bffbb4 2024-11-27T16:48:55,395 INFO [StoreOpener-4630e7dfe17ac910f657c7ced8cc5f5d-1 {}] regionserver.HStore(327): Store=4630e7dfe17ac910f657c7ced8cc5f5d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:55,396 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,396 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,398 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,398 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/recovered.edits/0000000000000000032 2024-11-27T16:48:55,400 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:48:55,402 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/recovered.edits/0000000000000000032 2024-11-27T16:48:55,402 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4630e7dfe17ac910f657c7ced8cc5f5d 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-27T16:48:55,420 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/.tmp/b/5cc45379bf01458cb91bd5bb3da99da6 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732726135165/Put/seqid=0 2024-11-27T16:48:55,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741900_1078 (size=5958) 2024-11-27T16:48:55,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741900_1078 (size=5958) 2024-11-27T16:48:55,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741900_1078 (size=5958) 2024-11-27T16:48:55,428 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/.tmp/b/5cc45379bf01458cb91bd5bb3da99da6 2024-11-27T16:48:55,434 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/.tmp/b/5cc45379bf01458cb91bd5bb3da99da6 as hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/b/5cc45379bf01458cb91bd5bb3da99da6 2024-11-27T16:48:55,440 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/b/5cc45379bf01458cb91bd5bb3da99da6, entries=10, sequenceid=32, filesize=5.8 K 2024-11-27T16:48:55,440 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 4630e7dfe17ac910f657c7ced8cc5f5d in 38ms, sequenceid=32, compaction requested=false; wal=null 2024-11-27T16:48:55,441 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/recovered.edits/0000000000000000032 2024-11-27T16:48:55,442 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,442 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,443 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T16:48:55,444 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4630e7dfe17ac910f657c7ced8cc5f5d 2024-11-27T16:48:55,447 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/4630e7dfe17ac910f657c7ced8cc5f5d/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-27T16:48:55,448 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4630e7dfe17ac910f657c7ced8cc5f5d; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66650719, jitterRate=-0.0068268924951553345}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T16:48:55,449 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4630e7dfe17ac910f657c7ced8cc5f5d: Writing region info on filesystem at 1732726135378Initializing all the Stores at 1732726135379 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726135379Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726135379Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726135379Obtaining lock to block concurrent updates at 1732726135402 (+23 ms)Preparing flush snapshotting stores in 4630e7dfe17ac910f657c7ced8cc5f5d at 1732726135402Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732726135062.4630e7dfe17ac910f657c7ced8cc5f5d., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732726135402Flushing stores of testReplayEditsWrittenViaHRegion,,1732726135062.4630e7dfe17ac910f657c7ced8cc5f5d. at 1732726135402Flushing 4630e7dfe17ac910f657c7ced8cc5f5d/b: creating writer at 1732726135402Flushing 4630e7dfe17ac910f657c7ced8cc5f5d/b: appending metadata at 1732726135419 (+17 ms)Flushing 4630e7dfe17ac910f657c7ced8cc5f5d/b: closing flushed file at 1732726135419Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f9e07af: reopening flushed file at 1732726135433 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 4630e7dfe17ac910f657c7ced8cc5f5d in 38ms, sequenceid=32, compaction requested=false; wal=null at 1732726135440 (+7 ms)Cleaning up temporary data from old regions at 1732726135442 (+2 ms)Region opened successfully at 1732726135449 (+7 ms) 2024-11-27T16:48:55,477 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=418 (was 408) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:46532 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:42496 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-89784467-172.17.0.2-1732726112257:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:53860 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:35332 [Waiting for operation #43] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-89784467-172.17.0.2-1732726112257:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:50564 [Waiting for operation #34] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:33910 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-89784467-172.17.0.2-1732726112257:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1122 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=273 (was 273), ProcessCount=11 (was 11), AvailableMemoryMB=6426 (was 6430) 2024-11-27T16:48:55,477 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1122 is superior to 1024 2024-11-27T16:48:55,492 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=418, OpenFileDescriptor=1122, MaxFileDescriptor=1048576, SystemLoadAverage=273, ProcessCount=11, AvailableMemoryMB=6426 2024-11-27T16:48:55,492 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1122 is superior to 1024 2024-11-27T16:48:55,507 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:55,509 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:48:55,510 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T16:48:55,513 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-50996258, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/hregion-50996258, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:55,526 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-50996258/hregion-50996258.1732726135513, exclude list is [], retry=0 2024-11-27T16:48:55,529 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:55,530 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:55,530 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:55,533 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-50996258/hregion-50996258.1732726135513 2024-11-27T16:48:55,533 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:39655:39655)] 2024-11-27T16:48:55,534 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 855906aef23f9fdc1b5d627955bdc050, NAME => 'testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40409/hbase 2024-11-27T16:48:55,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741902_1080 (size=68) 2024-11-27T16:48:55,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741902_1080 (size=68) 2024-11-27T16:48:55,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741902_1080 (size=68) 2024-11-27T16:48:55,542 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:55,548 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,549 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 855906aef23f9fdc1b5d627955bdc050 columnFamilyName a 2024-11-27T16:48:55,549 DEBUG [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:55,550 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] regionserver.HStore(327): Store=855906aef23f9fdc1b5d627955bdc050/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:55,550 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,551 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 855906aef23f9fdc1b5d627955bdc050 columnFamilyName b 2024-11-27T16:48:55,551 DEBUG [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:55,552 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] regionserver.HStore(327): Store=855906aef23f9fdc1b5d627955bdc050/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:55,552 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,553 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 855906aef23f9fdc1b5d627955bdc050 columnFamilyName c 2024-11-27T16:48:55,553 DEBUG [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:55,554 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] regionserver.HStore(327): Store=855906aef23f9fdc1b5d627955bdc050/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:55,554 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,554 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,555 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,556 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,556 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,556 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T16:48:55,557 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,559 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T16:48:55,560 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 855906aef23f9fdc1b5d627955bdc050; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74612670, jitterRate=0.1118154227733612}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T16:48:55,560 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 855906aef23f9fdc1b5d627955bdc050: Writing region info on filesystem at 1732726135543Initializing all the Stores at 1732726135543Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726135543Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726135548 (+5 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726135548Cleaning up temporary data from old regions at 1732726135556 (+8 ms)Region opened successfully at 1732726135560 (+4 ms) 2024-11-27T16:48:55,561 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 855906aef23f9fdc1b5d627955bdc050, disabling compactions & flushes 2024-11-27T16:48:55,561 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050. 2024-11-27T16:48:55,561 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050. 2024-11-27T16:48:55,561 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050. after waiting 0 ms 2024-11-27T16:48:55,561 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050. 2024-11-27T16:48:55,561 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050. 2024-11-27T16:48:55,561 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 855906aef23f9fdc1b5d627955bdc050: Waiting for close lock at 1732726135561Disabling compacts and flushes for region at 1732726135561Disabling writes for close at 1732726135561Writing region close event to WAL at 1732726135561Closed at 1732726135561 2024-11-27T16:48:55,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741901_1079 (size=95) 2024-11-27T16:48:55,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741901_1079 (size=95) 2024-11-27T16:48:55,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741901_1079 (size=95) 2024-11-27T16:48:55,566 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T16:48:55,566 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-50996258:(num 1732726135513) 2024-11-27T16:48:55,566 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:48:55,568 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:48:55,581 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506/wal.1732726135568, exclude list is [], retry=0 2024-11-27T16:48:55,584 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:48:55,584 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:48:55,585 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:48:55,586 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506/wal.1732726135568 2024-11-27T16:48:55,587 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:39655:39655)] 2024-11-27T16:48:55,639 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 855906aef23f9fdc1b5d627955bdc050, NAME => 'testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:48:55,641 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,642 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:48:55,642 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,642 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,644 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,645 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 855906aef23f9fdc1b5d627955bdc050 columnFamilyName a 2024-11-27T16:48:55,645 DEBUG [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:55,646 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] regionserver.HStore(327): Store=855906aef23f9fdc1b5d627955bdc050/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:55,646 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,647 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 855906aef23f9fdc1b5d627955bdc050 columnFamilyName b 2024-11-27T16:48:55,647 DEBUG [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:55,647 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] regionserver.HStore(327): Store=855906aef23f9fdc1b5d627955bdc050/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:55,647 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,648 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 855906aef23f9fdc1b5d627955bdc050 columnFamilyName c 2024-11-27T16:48:55,648 DEBUG [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:48:55,649 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] regionserver.HStore(327): Store=855906aef23f9fdc1b5d627955bdc050/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:48:55,649 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,650 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,651 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,652 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,652 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,652 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T16:48:55,654 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,654 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 855906aef23f9fdc1b5d627955bdc050; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61032554, jitterRate=-0.09054407477378845}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T16:48:55,654 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:48:55,655 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 855906aef23f9fdc1b5d627955bdc050: Running coprocessor pre-open hook at 1732726135642Writing region info on filesystem at 1732726135642Initializing all the Stores at 1732726135643 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726135643Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726135643Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726135643Cleaning up temporary data from old regions at 1732726135652 (+9 ms)Running coprocessor post-open hooks at 1732726135654 (+2 ms)Region opened successfully at 1732726135655 (+1 ms) 2024-11-27T16:48:55,670 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 855906aef23f9fdc1b5d627955bdc050 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-27T16:48:55,671 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 855906aef23f9fdc1b5d627955bdc050/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:48:56,043 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-27T16:48:56,043 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-27T16:48:56,045 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-27T16:48:56,045 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-27T16:48:56,045 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-27T16:48:56,045 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-27T16:48:56,046 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-27T16:48:56,046 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-27T16:48:56,047 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-27T16:48:56,047 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-27T16:48:56,672 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 855906aef23f9fdc1b5d627955bdc050/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:48:57,007 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T16:48:57,672 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 855906aef23f9fdc1b5d627955bdc050/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:48:58,673 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 855906aef23f9fdc1b5d627955bdc050/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:48:59,674 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 855906aef23f9fdc1b5d627955bdc050/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:49:00,674 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 855906aef23f9fdc1b5d627955bdc050/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:49:01,675 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 855906aef23f9fdc1b5d627955bdc050/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:49:02,676 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 855906aef23f9fdc1b5d627955bdc050/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:49:03,676 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 855906aef23f9fdc1b5d627955bdc050/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:49:04,677 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 855906aef23f9fdc1b5d627955bdc050/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:49:04,679 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 855906aef23f9fdc1b5d627955bdc050: 2024-11-27T16:49:04,679 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050. 2024-11-27T16:49:04,692 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T16:49:04,693 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 855906aef23f9fdc1b5d627955bdc050: 2024-11-27T16:49:04,693 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-27T16:49:04,693 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 855906aef23f9fdc1b5d627955bdc050, disabling compactions & flushes 2024-11-27T16:49:04,693 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050. 2024-11-27T16:49:04,693 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050. 2024-11-27T16:49:04,693 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050. after waiting 0 ms 2024-11-27T16:49:04,693 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050. 2024-11-27T16:49:04,694 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050. 2024-11-27T16:49:04,694 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050. 2024-11-27T16:49:04,694 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 855906aef23f9fdc1b5d627955bdc050: Waiting for close lock at 1732726144693Running coprocessor pre-close hooks at 1732726144693Disabling compacts and flushes for region at 1732726144693Disabling writes for close at 1732726144693Writing region close event to WAL at 1732726144693Running coprocessor post-close hooks at 1732726144694 (+1 ms)Closed at 1732726144694 2024-11-27T16:49:04,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741903_1081 (size=2691) 2024-11-27T16:49:04,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741903_1081 (size=2691) 2024-11-27T16:49:04,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741903_1081 (size=2691) 2024-11-27T16:49:04,713 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40409/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506/wal.1732726135568, size=2.6 K (2691bytes) 2024-11-27T16:49:04,714 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40409/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506/wal.1732726135568 2024-11-27T16:49:04,714 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40409/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506/wal.1732726135568 after 0ms 2024-11-27T16:49:04,716 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506/wal.1732726135568: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:49:04,717 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40409/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506/wal.1732726135568 took 4ms 2024-11-27T16:49:04,719 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40409/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506/wal.1732726135568 so closing down 2024-11-27T16:49:04,719 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T16:49:04,720 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732726135568.temp 2024-11-27T16:49:04,721 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/recovered.edits/0000000000000000004-wal.1732726135568.temp 2024-11-27T16:49:04,722 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T16:49:04,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741904_1082 (size=2094) 2024-11-27T16:49:04,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741904_1082 (size=2094) 2024-11-27T16:49:04,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741904_1082 (size=2094) 2024-11-27T16:49:04,735 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/recovered.edits/0000000000000000004-wal.1732726135568.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-27T16:49:04,737 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/recovered.edits/0000000000000000004-wal.1732726135568.temp to hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/recovered.edits/0000000000000000026 2024-11-27T16:49:04,737 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 20 ms; skipped=3; WAL=hdfs://localhost:40409/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506/wal.1732726135568, size=2.6 K, length=2691, corrupted=false, cancelled=false 2024-11-27T16:49:04,737 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40409/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506/wal.1732726135568, journal: Splitting hdfs://localhost:40409/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506/wal.1732726135568, size=2.6 K (2691bytes) at 1732726144713Finishing writing output for hdfs://localhost:40409/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506/wal.1732726135568 so closing down at 1732726144719 (+6 ms)Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/recovered.edits/0000000000000000004-wal.1732726135568.temp at 1732726144721 (+2 ms)3 split writer threads finished at 1732726144722 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/recovered.edits/0000000000000000004-wal.1732726135568.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732726144735 (+13 ms)Rename recovered edits hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/recovered.edits/0000000000000000004-wal.1732726135568.temp to hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/recovered.edits/0000000000000000026 at 1732726144737 (+2 ms)Processed 23 edits across 1 Regions in 20 ms; skipped=3; WAL=hdfs://localhost:40409/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506/wal.1732726135568, size=2.6 K, length=2691, corrupted=false, cancelled=false at 1732726144737 2024-11-27T16:49:04,739 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40409/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506/wal.1732726135568 to hdfs://localhost:40409/hbase/oldWALs/wal.1732726135568 2024-11-27T16:49:04,740 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/recovered.edits/0000000000000000026 2024-11-27T16:49:04,740 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:49:04,742 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:49:04,756 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506/wal.1732726144742, exclude list is [], retry=0 2024-11-27T16:49:04,759 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:49:04,759 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:49:04,759 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:49:04,763 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506/wal.1732726144742 2024-11-27T16:49:04,763 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:39655:39655)] 2024-11-27T16:49:04,764 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 855906aef23f9fdc1b5d627955bdc050, NAME => 'testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:49:04,765 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:49:04,765 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:49:04,765 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:49:04,765 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:49:04,768 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:49:04,769 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 855906aef23f9fdc1b5d627955bdc050 columnFamilyName a 2024-11-27T16:49:04,769 DEBUG [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:04,770 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] regionserver.HStore(327): Store=855906aef23f9fdc1b5d627955bdc050/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:04,770 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:49:04,771 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 855906aef23f9fdc1b5d627955bdc050 columnFamilyName b 2024-11-27T16:49:04,771 DEBUG [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:04,772 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] regionserver.HStore(327): Store=855906aef23f9fdc1b5d627955bdc050/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:04,772 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:49:04,773 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 855906aef23f9fdc1b5d627955bdc050 columnFamilyName c 2024-11-27T16:49:04,773 DEBUG [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:04,773 INFO [StoreOpener-855906aef23f9fdc1b5d627955bdc050-1 {}] regionserver.HStore(327): Store=855906aef23f9fdc1b5d627955bdc050/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:04,773 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:49:04,774 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:49:04,775 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:49:04,776 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/recovered.edits/0000000000000000026 2024-11-27T16:49:04,778 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:49:04,781 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/recovered.edits/0000000000000000026 2024-11-27T16:49:04,781 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 855906aef23f9fdc1b5d627955bdc050 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-27T16:49:04,804 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/.tmp/a/340593f833a04110a34b5f4683f444aa is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732726144685/Put/seqid=0 2024-11-27T16:49:04,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741906_1084 (size=5523) 2024-11-27T16:49:04,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741906_1084 (size=5523) 2024-11-27T16:49:04,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741906_1084 (size=5523) 2024-11-27T16:49:04,816 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/.tmp/a/340593f833a04110a34b5f4683f444aa 2024-11-27T16:49:04,846 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/.tmp/b/341ffc42243a4bb7b25ff76e27793219 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732726144679/Put/seqid=0 2024-11-27T16:49:04,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741907_1085 (size=5524) 2024-11-27T16:49:04,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741907_1085 (size=5524) 2024-11-27T16:49:04,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741907_1085 (size=5524) 2024-11-27T16:49:04,857 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/.tmp/b/341ffc42243a4bb7b25ff76e27793219 2024-11-27T16:49:04,887 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/.tmp/c/7d2cb3d5746240d3aa3ca4650b2cce74 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732726144682/Put/seqid=0 2024-11-27T16:49:04,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741908_1086 (size=5457) 2024-11-27T16:49:04,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741908_1086 (size=5457) 2024-11-27T16:49:04,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741908_1086 (size=5457) 2024-11-27T16:49:04,897 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/.tmp/c/7d2cb3d5746240d3aa3ca4650b2cce74 2024-11-27T16:49:04,905 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/.tmp/a/340593f833a04110a34b5f4683f444aa as hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/a/340593f833a04110a34b5f4683f444aa 2024-11-27T16:49:04,911 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/a/340593f833a04110a34b5f4683f444aa, entries=7, sequenceid=26, filesize=5.4 K 2024-11-27T16:49:04,913 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/.tmp/b/341ffc42243a4bb7b25ff76e27793219 as hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/b/341ffc42243a4bb7b25ff76e27793219 2024-11-27T16:49:04,919 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/b/341ffc42243a4bb7b25ff76e27793219, entries=7, sequenceid=26, filesize=5.4 K 2024-11-27T16:49:04,921 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/.tmp/c/7d2cb3d5746240d3aa3ca4650b2cce74 as hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/c/7d2cb3d5746240d3aa3ca4650b2cce74 2024-11-27T16:49:04,927 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/c/7d2cb3d5746240d3aa3ca4650b2cce74, entries=6, sequenceid=26, filesize=5.3 K 2024-11-27T16:49:04,928 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 855906aef23f9fdc1b5d627955bdc050 in 146ms, sequenceid=26, compaction requested=false; wal=null 2024-11-27T16:49:04,929 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/recovered.edits/0000000000000000026 2024-11-27T16:49:04,930 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:49:04,930 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:49:04,931 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T16:49:04,933 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:49:04,935 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testReplayEditsAfterAbortingFlush/855906aef23f9fdc1b5d627955bdc050/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-27T16:49:04,937 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 855906aef23f9fdc1b5d627955bdc050; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63945379, jitterRate=-0.04713959991931915}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T16:49:04,937 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 855906aef23f9fdc1b5d627955bdc050 2024-11-27T16:49:04,938 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 855906aef23f9fdc1b5d627955bdc050: Running coprocessor pre-open hook at 1732726144765Writing region info on filesystem at 1732726144765Initializing all the Stores at 1732726144767 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726144767Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726144767Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726144767Obtaining lock to block concurrent updates at 1732726144781 (+14 ms)Preparing flush snapshotting stores in 855906aef23f9fdc1b5d627955bdc050 at 1732726144781Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732726144781Flushing stores of testReplayEditsAfterAbortingFlush,,1732726135508.855906aef23f9fdc1b5d627955bdc050. at 1732726144781Flushing 855906aef23f9fdc1b5d627955bdc050/a: creating writer at 1732726144781Flushing 855906aef23f9fdc1b5d627955bdc050/a: appending metadata at 1732726144803 (+22 ms)Flushing 855906aef23f9fdc1b5d627955bdc050/a: closing flushed file at 1732726144803Flushing 855906aef23f9fdc1b5d627955bdc050/b: creating writer at 1732726144822 (+19 ms)Flushing 855906aef23f9fdc1b5d627955bdc050/b: appending metadata at 1732726144845 (+23 ms)Flushing 855906aef23f9fdc1b5d627955bdc050/b: closing flushed file at 1732726144845Flushing 855906aef23f9fdc1b5d627955bdc050/c: creating writer at 1732726144864 (+19 ms)Flushing 855906aef23f9fdc1b5d627955bdc050/c: appending metadata at 1732726144886 (+22 ms)Flushing 855906aef23f9fdc1b5d627955bdc050/c: closing flushed file at 1732726144886Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4edf61db: reopening flushed file at 1732726144904 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cede39f: reopening flushed file at 1732726144912 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6da719ff: reopening flushed file at 1732726144920 (+8 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 855906aef23f9fdc1b5d627955bdc050 in 146ms, sequenceid=26, compaction requested=false; wal=null at 1732726144928 (+8 ms)Cleaning up temporary data from old regions at 1732726144930 (+2 ms)Running coprocessor post-open hooks at 1732726144937 (+7 ms)Region opened successfully at 1732726144937 2024-11-27T16:49:04,962 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=416 (was 418), OpenFileDescriptor=1180 (was 1122) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=247 (was 273), ProcessCount=11 (was 11), AvailableMemoryMB=6395 (was 6426) 2024-11-27T16:49:04,962 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-11-27T16:49:04,976 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=416, OpenFileDescriptor=1180, MaxFileDescriptor=1048576, SystemLoadAverage=247, ProcessCount=11, AvailableMemoryMB=6390 2024-11-27T16:49:04,976 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-11-27T16:49:04,999 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:49:05,002 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:49:05,003 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T16:49:05,005 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-92593787, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/hregion-92593787, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:49:05,026 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-92593787/hregion-92593787.1732726145006, exclude list is [], retry=0 2024-11-27T16:49:05,029 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:49:05,029 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:49:05,031 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:49:05,034 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-92593787/hregion-92593787.1732726145006 2024-11-27T16:49:05,035 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:33829:33829)] 2024-11-27T16:49:05,035 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 529c0c9d8ce3b4834c1086be6fa2123e, NAME => 'testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40409/hbase 2024-11-27T16:49:05,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741910_1088 (size=61) 2024-11-27T16:49:05,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741910_1088 (size=61) 2024-11-27T16:49:05,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741910_1088 (size=61) 2024-11-27T16:49:05,048 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:49:05,050 INFO [StoreOpener-529c0c9d8ce3b4834c1086be6fa2123e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,051 INFO [StoreOpener-529c0c9d8ce3b4834c1086be6fa2123e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 529c0c9d8ce3b4834c1086be6fa2123e columnFamilyName a 2024-11-27T16:49:05,051 DEBUG [StoreOpener-529c0c9d8ce3b4834c1086be6fa2123e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:05,052 INFO [StoreOpener-529c0c9d8ce3b4834c1086be6fa2123e-1 {}] regionserver.HStore(327): Store=529c0c9d8ce3b4834c1086be6fa2123e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:05,052 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,053 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,053 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,054 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,054 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,055 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,058 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T16:49:05,058 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 529c0c9d8ce3b4834c1086be6fa2123e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71378962, jitterRate=0.06362941861152649}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T16:49:05,059 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 529c0c9d8ce3b4834c1086be6fa2123e: Writing region info on filesystem at 1732726145048Initializing all the Stores at 1732726145049 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726145049Cleaning up temporary data from old regions at 1732726145054 (+5 ms)Region opened successfully at 1732726145059 (+5 ms) 2024-11-27T16:49:05,059 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 529c0c9d8ce3b4834c1086be6fa2123e, disabling compactions & flushes 2024-11-27T16:49:05,059 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e. 2024-11-27T16:49:05,059 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e. 2024-11-27T16:49:05,059 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e. after waiting 0 ms 2024-11-27T16:49:05,059 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e. 2024-11-27T16:49:05,060 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e. 2024-11-27T16:49:05,060 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 529c0c9d8ce3b4834c1086be6fa2123e: Waiting for close lock at 1732726145059Disabling compacts and flushes for region at 1732726145059Disabling writes for close at 1732726145059Writing region close event to WAL at 1732726145060 (+1 ms)Closed at 1732726145060 2024-11-27T16:49:05,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741909_1087 (size=95) 2024-11-27T16:49:05,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741909_1087 (size=95) 2024-11-27T16:49:05,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741909_1087 (size=95) 2024-11-27T16:49:05,065 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T16:49:05,065 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-92593787:(num 1732726145006) 2024-11-27T16:49:05,065 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:49:05,067 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:49:05,082 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999/wal.1732726145067, exclude list is [], retry=0 2024-11-27T16:49:05,085 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:49:05,085 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:49:05,086 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:49:05,088 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999/wal.1732726145067 2024-11-27T16:49:05,089 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:42883:42883)] 2024-11-27T16:49:05,090 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 529c0c9d8ce3b4834c1086be6fa2123e, NAME => 'testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:49:05,090 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:49:05,090 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,090 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,092 INFO [StoreOpener-529c0c9d8ce3b4834c1086be6fa2123e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,093 INFO [StoreOpener-529c0c9d8ce3b4834c1086be6fa2123e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 529c0c9d8ce3b4834c1086be6fa2123e columnFamilyName a 2024-11-27T16:49:05,093 DEBUG [StoreOpener-529c0c9d8ce3b4834c1086be6fa2123e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:05,094 INFO [StoreOpener-529c0c9d8ce3b4834c1086be6fa2123e-1 {}] regionserver.HStore(327): Store=529c0c9d8ce3b4834c1086be6fa2123e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:05,094 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,095 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,096 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,096 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,096 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,099 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,100 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 529c0c9d8ce3b4834c1086be6fa2123e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62770264, jitterRate=-0.06465017795562744}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T16:49:05,101 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 529c0c9d8ce3b4834c1086be6fa2123e: Writing region info on filesystem at 1732726145090Initializing all the Stores at 1732726145091 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726145091Cleaning up temporary data from old regions at 1732726145097 (+6 ms)Region opened successfully at 1732726145101 (+4 ms) 2024-11-27T16:49:05,113 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 529c0c9d8ce3b4834c1086be6fa2123e, disabling compactions & flushes 2024-11-27T16:49:05,113 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e. 2024-11-27T16:49:05,113 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e. 2024-11-27T16:49:05,113 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e. after waiting 0 ms 2024-11-27T16:49:05,113 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e. 2024-11-27T16:49:05,114 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e. 2024-11-27T16:49:05,114 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e. 2024-11-27T16:49:05,114 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 529c0c9d8ce3b4834c1086be6fa2123e: Waiting for close lock at 1732726145113Disabling compacts and flushes for region at 1732726145113Disabling writes for close at 1732726145113Writing region close event to WAL at 1732726145114 (+1 ms)Closed at 1732726145114 2024-11-27T16:49:05,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741911_1089 (size=1050) 2024-11-27T16:49:05,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741911_1089 (size=1050) 2024-11-27T16:49:05,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741911_1089 (size=1050) 2024-11-27T16:49:05,145 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40409/hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999/wal.1732726145067, size=1.0 K (1050bytes) 2024-11-27T16:49:05,145 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40409/hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999/wal.1732726145067 2024-11-27T16:49:05,146 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40409/hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999/wal.1732726145067 after 1ms 2024-11-27T16:49:05,148 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999/wal.1732726145067: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:49:05,148 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40409/hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999/wal.1732726145067 took 3ms 2024-11-27T16:49:05,150 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40409/hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999/wal.1732726145067 so closing down 2024-11-27T16:49:05,151 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T16:49:05,152 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732726145067.temp 2024-11-27T16:49:05,154 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/recovered.edits/0000000000000000003-wal.1732726145067.temp 2024-11-27T16:49:05,154 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T16:49:05,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741912_1090 (size=1050) 2024-11-27T16:49:05,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741912_1090 (size=1050) 2024-11-27T16:49:05,162 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/recovered.edits/0000000000000000003-wal.1732726145067.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-27T16:49:05,164 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/recovered.edits/0000000000000000003-wal.1732726145067.temp to hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/recovered.edits/0000000000000000012 2024-11-27T16:49:05,164 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:40409/hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999/wal.1732726145067, size=1.0 K, length=1050, corrupted=false, cancelled=false 2024-11-27T16:49:05,164 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40409/hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999/wal.1732726145067, journal: Splitting hdfs://localhost:40409/hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999/wal.1732726145067, size=1.0 K (1050bytes) at 1732726145145Finishing writing output for hdfs://localhost:40409/hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999/wal.1732726145067 so closing down at 1732726145150 (+5 ms)Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/recovered.edits/0000000000000000003-wal.1732726145067.temp at 1732726145154 (+4 ms)3 split writer threads finished at 1732726145154Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/recovered.edits/0000000000000000003-wal.1732726145067.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732726145162 (+8 ms)Rename recovered edits hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/recovered.edits/0000000000000000003-wal.1732726145067.temp to hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/recovered.edits/0000000000000000012 at 1732726145164 (+2 ms)Processed 10 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:40409/hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999/wal.1732726145067, size=1.0 K, length=1050, corrupted=false, cancelled=false at 1732726145164 2024-11-27T16:49:05,166 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40409/hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999/wal.1732726145067 to hdfs://localhost:40409/hbase/oldWALs/wal.1732726145067 2024-11-27T16:49:05,167 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/recovered.edits/0000000000000000012 2024-11-27T16:49:05,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741912_1090 (size=1050) 2024-11-27T16:49:05,171 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:49:05,582 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:49:05,585 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:49:05,599 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999/wal.1732726145585, exclude list is [], retry=0 2024-11-27T16:49:05,602 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:49:05,603 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:49:05,603 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:49:05,605 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999/wal.1732726145585 2024-11-27T16:49:05,608 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:33829:33829)] 2024-11-27T16:49:05,608 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 529c0c9d8ce3b4834c1086be6fa2123e, NAME => 'testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:49:05,608 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:49:05,608 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,608 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,612 INFO [StoreOpener-529c0c9d8ce3b4834c1086be6fa2123e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,614 INFO [StoreOpener-529c0c9d8ce3b4834c1086be6fa2123e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 529c0c9d8ce3b4834c1086be6fa2123e columnFamilyName a 2024-11-27T16:49:05,614 DEBUG [StoreOpener-529c0c9d8ce3b4834c1086be6fa2123e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:05,615 INFO [StoreOpener-529c0c9d8ce3b4834c1086be6fa2123e-1 {}] regionserver.HStore(327): Store=529c0c9d8ce3b4834c1086be6fa2123e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:05,616 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,616 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,618 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,618 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/recovered.edits/0000000000000000012 2024-11-27T16:49:05,621 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:49:05,622 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/recovered.edits/0000000000000000012 2024-11-27T16:49:05,622 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 529c0c9d8ce3b4834c1086be6fa2123e 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-27T16:49:05,643 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/.tmp/a/3ef22586376d4d4dbaf67d4cba21804e is 79, key is testDatalossWhenInputError/a:x0/1732726145101/Put/seqid=0 2024-11-27T16:49:05,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741914_1092 (size=5808) 2024-11-27T16:49:05,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741914_1092 (size=5808) 2024-11-27T16:49:05,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741914_1092 (size=5808) 2024-11-27T16:49:05,652 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/.tmp/a/3ef22586376d4d4dbaf67d4cba21804e 2024-11-27T16:49:05,662 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/.tmp/a/3ef22586376d4d4dbaf67d4cba21804e as hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/a/3ef22586376d4d4dbaf67d4cba21804e 2024-11-27T16:49:05,672 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/a/3ef22586376d4d4dbaf67d4cba21804e, entries=10, sequenceid=12, filesize=5.7 K 2024-11-27T16:49:05,673 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 529c0c9d8ce3b4834c1086be6fa2123e in 51ms, sequenceid=12, compaction requested=false; wal=null 2024-11-27T16:49:05,673 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/recovered.edits/0000000000000000012 2024-11-27T16:49:05,674 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,674 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,677 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,679 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-27T16:49:05,680 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 529c0c9d8ce3b4834c1086be6fa2123e; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66856077, jitterRate=-0.0037668198347091675}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T16:49:05,681 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 529c0c9d8ce3b4834c1086be6fa2123e: Writing region info on filesystem at 1732726145608Initializing all the Stores at 1732726145611 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726145611Obtaining lock to block concurrent updates at 1732726145622 (+11 ms)Preparing flush snapshotting stores in 529c0c9d8ce3b4834c1086be6fa2123e at 1732726145622Finished memstore snapshotting testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732726145622Flushing stores of testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e. at 1732726145622Flushing 529c0c9d8ce3b4834c1086be6fa2123e/a: creating writer at 1732726145622Flushing 529c0c9d8ce3b4834c1086be6fa2123e/a: appending metadata at 1732726145642 (+20 ms)Flushing 529c0c9d8ce3b4834c1086be6fa2123e/a: closing flushed file at 1732726145642Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30a741bb: reopening flushed file at 1732726145660 (+18 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 529c0c9d8ce3b4834c1086be6fa2123e in 51ms, sequenceid=12, compaction requested=false; wal=null at 1732726145673 (+13 ms)Cleaning up temporary data from old regions at 1732726145674 (+1 ms)Region opened successfully at 1732726145681 (+7 ms) 2024-11-27T16:49:05,684 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 529c0c9d8ce3b4834c1086be6fa2123e, NAME => 'testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:49:05,684 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732726145000.529c0c9d8ce3b4834c1086be6fa2123e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:49:05,685 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,685 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,686 INFO [StoreOpener-529c0c9d8ce3b4834c1086be6fa2123e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,687 INFO [StoreOpener-529c0c9d8ce3b4834c1086be6fa2123e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 529c0c9d8ce3b4834c1086be6fa2123e columnFamilyName a 2024-11-27T16:49:05,687 DEBUG [StoreOpener-529c0c9d8ce3b4834c1086be6fa2123e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:05,693 DEBUG [StoreOpener-529c0c9d8ce3b4834c1086be6fa2123e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/a/3ef22586376d4d4dbaf67d4cba21804e 2024-11-27T16:49:05,693 INFO [StoreOpener-529c0c9d8ce3b4834c1086be6fa2123e-1 {}] regionserver.HStore(327): Store=529c0c9d8ce3b4834c1086be6fa2123e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:05,694 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,694 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,696 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,696 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,696 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,698 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 529c0c9d8ce3b4834c1086be6fa2123e 2024-11-27T16:49:05,701 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testDatalossWhenInputError/529c0c9d8ce3b4834c1086be6fa2123e/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-27T16:49:05,702 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 529c0c9d8ce3b4834c1086be6fa2123e; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66356601, jitterRate=-0.01120959222316742}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T16:49:05,702 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 529c0c9d8ce3b4834c1086be6fa2123e: Writing region info on filesystem at 1732726145685Initializing all the Stores at 1732726145685Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726145685Cleaning up temporary data from old regions at 1732726145696 (+11 ms)Region opened successfully at 1732726145702 (+6 ms) 2024-11-27T16:49:05,723 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=425 (was 416) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:44984 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:43752 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:45074 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-89784467-172.17.0.2-1732726112257:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:43808 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:49148 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:49194 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-89784467-172.17.0.2-1732726112257:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-89784467-172.17.0.2-1732726112257:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1262 (was 1180) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=247 (was 247), ProcessCount=11 (was 11), AvailableMemoryMB=6374 (was 6390) 2024-11-27T16:49:05,723 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-11-27T16:49:05,738 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=425, OpenFileDescriptor=1262, MaxFileDescriptor=1048576, SystemLoadAverage=247, ProcessCount=11, AvailableMemoryMB=6374 2024-11-27T16:49:05,738 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-11-27T16:49:05,754 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:49:05,755 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:49:05,756 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T16:49:05,758 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-10232086, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/hregion-10232086, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:49:05,772 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-10232086/hregion-10232086.1732726145759, exclude list is [], retry=0 2024-11-27T16:49:05,775 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:49:05,775 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:49:05,775 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:49:05,777 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-10232086/hregion-10232086.1732726145759 2024-11-27T16:49:05,777 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:33829:33829)] 2024-11-27T16:49:05,778 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a26e33e0aaac41ddc6329f0f1cbe48d7, NAME => 'testCompactedBulkLoadedFiles,,1732726145754.a26e33e0aaac41ddc6329f0f1cbe48d7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40409/hbase 2024-11-27T16:49:05,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741916_1094 (size=63) 2024-11-27T16:49:05,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741916_1094 (size=63) 2024-11-27T16:49:05,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741916_1094 (size=63) 2024-11-27T16:49:05,791 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732726145754.a26e33e0aaac41ddc6329f0f1cbe48d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:49:05,792 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,794 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a26e33e0aaac41ddc6329f0f1cbe48d7 columnFamilyName a 2024-11-27T16:49:05,794 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:05,794 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.HStore(327): Store=a26e33e0aaac41ddc6329f0f1cbe48d7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:05,794 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,795 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a26e33e0aaac41ddc6329f0f1cbe48d7 columnFamilyName b 2024-11-27T16:49:05,796 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:05,796 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.HStore(327): Store=a26e33e0aaac41ddc6329f0f1cbe48d7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:05,796 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,797 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a26e33e0aaac41ddc6329f0f1cbe48d7 columnFamilyName c 2024-11-27T16:49:05,797 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:05,798 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.HStore(327): Store=a26e33e0aaac41ddc6329f0f1cbe48d7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:05,798 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,799 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,799 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,801 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,801 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,801 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T16:49:05,802 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,804 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T16:49:05,805 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a26e33e0aaac41ddc6329f0f1cbe48d7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72043220, jitterRate=0.07352763414382935}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T16:49:05,806 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a26e33e0aaac41ddc6329f0f1cbe48d7: Writing region info on filesystem at 1732726145791Initializing all the Stores at 1732726145792 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726145792Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726145792Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726145792Cleaning up temporary data from old regions at 1732726145801 (+9 ms)Region opened successfully at 1732726145806 (+5 ms) 2024-11-27T16:49:05,806 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a26e33e0aaac41ddc6329f0f1cbe48d7, disabling compactions & flushes 2024-11-27T16:49:05,806 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732726145754.a26e33e0aaac41ddc6329f0f1cbe48d7. 2024-11-27T16:49:05,806 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732726145754.a26e33e0aaac41ddc6329f0f1cbe48d7. 2024-11-27T16:49:05,806 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732726145754.a26e33e0aaac41ddc6329f0f1cbe48d7. after waiting 0 ms 2024-11-27T16:49:05,806 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732726145754.a26e33e0aaac41ddc6329f0f1cbe48d7. 2024-11-27T16:49:05,806 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732726145754.a26e33e0aaac41ddc6329f0f1cbe48d7. 2024-11-27T16:49:05,806 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a26e33e0aaac41ddc6329f0f1cbe48d7: Waiting for close lock at 1732726145806Disabling compacts and flushes for region at 1732726145806Disabling writes for close at 1732726145806Writing region close event to WAL at 1732726145806Closed at 1732726145806 2024-11-27T16:49:05,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741915_1093 (size=95) 2024-11-27T16:49:05,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741915_1093 (size=95) 2024-11-27T16:49:05,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741915_1093 (size=95) 2024-11-27T16:49:05,819 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T16:49:05,819 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-10232086:(num 1732726145759) 2024-11-27T16:49:05,819 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:49:05,821 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:49:05,835 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726145821, exclude list is [], retry=0 2024-11-27T16:49:05,837 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:49:05,838 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:49:05,838 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:49:05,842 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726145821 2024-11-27T16:49:05,844 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:33829:33829)] 2024-11-27T16:49:05,844 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a26e33e0aaac41ddc6329f0f1cbe48d7, NAME => 'testCompactedBulkLoadedFiles,,1732726145754.a26e33e0aaac41ddc6329f0f1cbe48d7.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:49:05,844 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732726145754.a26e33e0aaac41ddc6329f0f1cbe48d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:49:05,844 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,844 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,848 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,849 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a26e33e0aaac41ddc6329f0f1cbe48d7 columnFamilyName a 2024-11-27T16:49:05,849 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:05,849 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.HStore(327): Store=a26e33e0aaac41ddc6329f0f1cbe48d7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:05,849 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,850 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a26e33e0aaac41ddc6329f0f1cbe48d7 columnFamilyName b 2024-11-27T16:49:05,851 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:05,851 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.HStore(327): Store=a26e33e0aaac41ddc6329f0f1cbe48d7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:05,851 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,852 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a26e33e0aaac41ddc6329f0f1cbe48d7 columnFamilyName c 2024-11-27T16:49:05,852 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:05,853 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.HStore(327): Store=a26e33e0aaac41ddc6329f0f1cbe48d7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:05,853 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,854 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,855 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,856 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,856 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,857 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T16:49:05,858 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:05,859 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a26e33e0aaac41ddc6329f0f1cbe48d7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64161255, jitterRate=-0.043922796845436096}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T16:49:05,860 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a26e33e0aaac41ddc6329f0f1cbe48d7: Writing region info on filesystem at 1732726145845Initializing all the Stores at 1732726145846 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726145846Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726145847 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726145847Cleaning up temporary data from old regions at 1732726145856 (+9 ms)Region opened successfully at 1732726145860 (+4 ms) 2024-11-27T16:49:05,864 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732726145864/Put/seqid=0 2024-11-27T16:49:05,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741918_1096 (size=4875) 2024-11-27T16:49:05,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741918_1096 (size=4875) 2024-11-27T16:49:05,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741918_1096 (size=4875) 2024-11-27T16:49:05,881 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732726145881/Put/seqid=0 2024-11-27T16:49:05,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741919_1097 (size=4875) 2024-11-27T16:49:05,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741919_1097 (size=4875) 2024-11-27T16:49:05,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741919_1097 (size=4875) 2024-11-27T16:49:05,895 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732726145895/Put/seqid=0 2024-11-27T16:49:05,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741920_1098 (size=4875) 2024-11-27T16:49:05,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741920_1098 (size=4875) 2024-11-27T16:49:05,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741920_1098 (size=4875) 2024-11-27T16:49:05,902 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40409/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in a26e33e0aaac41ddc6329f0f1cbe48d7/a 2024-11-27T16:49:05,907 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-27T16:49:05,907 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-27T16:49:05,907 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40409/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in a26e33e0aaac41ddc6329f0f1cbe48d7/a 2024-11-27T16:49:05,911 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-27T16:49:05,911 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-27T16:49:05,911 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40409/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in a26e33e0aaac41ddc6329f0f1cbe48d7/a 2024-11-27T16:49:05,915 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-27T16:49:05,915 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-27T16:49:05,915 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a26e33e0aaac41ddc6329f0f1cbe48d7 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-27T16:49:05,938 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/.tmp/a/be8f070fefbb40db8e98b2870006ba3c is 55, key is testCompactedBulkLoadedFiles/a:a/1732726145860/Put/seqid=0 2024-11-27T16:49:05,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741921_1099 (size=5107) 2024-11-27T16:49:05,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741921_1099 (size=5107) 2024-11-27T16:49:05,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741921_1099 (size=5107) 2024-11-27T16:49:05,950 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/.tmp/a/be8f070fefbb40db8e98b2870006ba3c 2024-11-27T16:49:05,957 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/.tmp/a/be8f070fefbb40db8e98b2870006ba3c as hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/be8f070fefbb40db8e98b2870006ba3c 2024-11-27T16:49:05,962 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/be8f070fefbb40db8e98b2870006ba3c, entries=1, sequenceid=4, filesize=5.0 K 2024-11-27T16:49:05,963 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for a26e33e0aaac41ddc6329f0f1cbe48d7 in 48ms, sequenceid=4, compaction requested=false 2024-11-27T16:49:05,963 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a26e33e0aaac41ddc6329f0f1cbe48d7: 2024-11-27T16:49:05,965 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/62b9c6b0e7b74a1b9da8daa5190a0f14_SeqId_4_ 2024-11-27T16:49:05,966 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/01e469d7302745c0b12122d7f5b49e4b_SeqId_4_ 2024-11-27T16:49:05,967 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/35fce9a667db4b7f9affad437508c9bd_SeqId_4_ 2024-11-27T16:49:05,968 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40409/hbase/testCompactedBulkLoadedFiles/hfile0 into a26e33e0aaac41ddc6329f0f1cbe48d7/a as hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/62b9c6b0e7b74a1b9da8daa5190a0f14_SeqId_4_ - updating store file list. 2024-11-27T16:49:05,972 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 62b9c6b0e7b74a1b9da8daa5190a0f14_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T16:49:05,973 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/62b9c6b0e7b74a1b9da8daa5190a0f14_SeqId_4_ into a26e33e0aaac41ddc6329f0f1cbe48d7/a 2024-11-27T16:49:05,973 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40409/hbase/testCompactedBulkLoadedFiles/hfile0 into a26e33e0aaac41ddc6329f0f1cbe48d7/a (new location: hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/62b9c6b0e7b74a1b9da8daa5190a0f14_SeqId_4_) 2024-11-27T16:49:05,974 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40409/hbase/testCompactedBulkLoadedFiles/hfile1 into a26e33e0aaac41ddc6329f0f1cbe48d7/a as hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/01e469d7302745c0b12122d7f5b49e4b_SeqId_4_ - updating store file list. 2024-11-27T16:49:05,978 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 01e469d7302745c0b12122d7f5b49e4b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T16:49:05,979 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/01e469d7302745c0b12122d7f5b49e4b_SeqId_4_ into a26e33e0aaac41ddc6329f0f1cbe48d7/a 2024-11-27T16:49:05,979 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40409/hbase/testCompactedBulkLoadedFiles/hfile1 into a26e33e0aaac41ddc6329f0f1cbe48d7/a (new location: hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/01e469d7302745c0b12122d7f5b49e4b_SeqId_4_) 2024-11-27T16:49:05,980 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40409/hbase/testCompactedBulkLoadedFiles/hfile2 into a26e33e0aaac41ddc6329f0f1cbe48d7/a as hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/35fce9a667db4b7f9affad437508c9bd_SeqId_4_ - updating store file list. 2024-11-27T16:49:05,985 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 35fce9a667db4b7f9affad437508c9bd_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T16:49:05,985 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/35fce9a667db4b7f9affad437508c9bd_SeqId_4_ into a26e33e0aaac41ddc6329f0f1cbe48d7/a 2024-11-27T16:49:05,985 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40409/hbase/testCompactedBulkLoadedFiles/hfile2 into a26e33e0aaac41ddc6329f0f1cbe48d7/a (new location: hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/35fce9a667db4b7f9affad437508c9bd_SeqId_4_) 2024-11-27T16:49:05,993 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T16:49:05,993 DEBUG [Time-limited test {}] regionserver.HStore(1541): a26e33e0aaac41ddc6329f0f1cbe48d7/a is initiating major compaction (all files) 2024-11-27T16:49:05,993 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of a26e33e0aaac41ddc6329f0f1cbe48d7/a in testCompactedBulkLoadedFiles,,1732726145754.a26e33e0aaac41ddc6329f0f1cbe48d7. 2024-11-27T16:49:05,993 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/be8f070fefbb40db8e98b2870006ba3c, hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/62b9c6b0e7b74a1b9da8daa5190a0f14_SeqId_4_, hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/01e469d7302745c0b12122d7f5b49e4b_SeqId_4_, hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/35fce9a667db4b7f9affad437508c9bd_SeqId_4_] into tmpdir=hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/.tmp, totalSize=19.3 K 2024-11-27T16:49:05,994 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting be8f070fefbb40db8e98b2870006ba3c, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732726145860 2024-11-27T16:49:05,995 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 62b9c6b0e7b74a1b9da8daa5190a0f14_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-27T16:49:05,995 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 01e469d7302745c0b12122d7f5b49e4b_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-27T16:49:05,996 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 35fce9a667db4b7f9affad437508c9bd_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-27T16:49:06,011 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/.tmp/a/213df5ba45fa45cbbd4bb108a500231c is 55, key is testCompactedBulkLoadedFiles/a:a/1732726145860/Put/seqid=0 2024-11-27T16:49:06,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741922_1100 (size=6154) 2024-11-27T16:49:06,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741922_1100 (size=6154) 2024-11-27T16:49:06,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741922_1100 (size=6154) 2024-11-27T16:49:06,035 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/.tmp/a/213df5ba45fa45cbbd4bb108a500231c as hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/213df5ba45fa45cbbd4bb108a500231c 2024-11-27T16:49:06,043 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in a26e33e0aaac41ddc6329f0f1cbe48d7/a of a26e33e0aaac41ddc6329f0f1cbe48d7 into 213df5ba45fa45cbbd4bb108a500231c(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T16:49:06,043 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for a26e33e0aaac41ddc6329f0f1cbe48d7: 2024-11-27T16:49:06,043 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-27T16:49:06,043 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-27T16:49:06,043 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-27T16:49:06,043 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-27T16:49:06,045 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-27T16:49:06,045 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-27T16:49:06,092 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40409/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726145821, size=0 (0bytes) 2024-11-27T16:49:06,092 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40409/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726145821 might be still open, length is 0 2024-11-27T16:49:06,092 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40409/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726145821 2024-11-27T16:49:06,093 WARN [IPC Server handler 0 on default port 40409 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726145821 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-27T16:49:06,093 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40409/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726145821 after 1ms 2024-11-27T16:49:07,466 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:45110 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:37433:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45110 dst: /127.0.0.1:37433 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37433 remote=/127.0.0.1:45110]. Total timeout mills is 60000, 58578 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:49:07,466 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:43838 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:41739:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43838 dst: /127.0.0.1:41739 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:49:07,466 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:49232 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:37511:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49232 dst: /127.0.0.1:37511 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:49:07,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741917_1101 (size=1173) 2024-11-27T16:49:07,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741917_1101 (size=1173) 2024-11-27T16:49:10,094 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40409/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726145821 after 4002ms 2024-11-27T16:49:10,097 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726145821: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:49:10,097 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40409/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726145821 took 4005ms 2024-11-27T16:49:10,100 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40409/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726145821; continuing. 2024-11-27T16:49:10,100 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40409/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726145821 so closing down 2024-11-27T16:49:10,100 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T16:49:10,101 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732726145821.temp 2024-11-27T16:49:10,102 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/recovered.edits/0000000000000000003-wal.1732726145821.temp 2024-11-27T16:49:10,103 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T16:49:10,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741923_1102 (size=548) 2024-11-27T16:49:10,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741923_1102 (size=548) 2024-11-27T16:49:10,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741923_1102 (size=548) 2024-11-27T16:49:10,109 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/recovered.edits/0000000000000000003-wal.1732726145821.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-27T16:49:10,111 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/recovered.edits/0000000000000000003-wal.1732726145821.temp to hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/recovered.edits/0000000000000000008 2024-11-27T16:49:10,111 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:40409/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726145821, size=0, length=0, corrupted=false, cancelled=false 2024-11-27T16:49:10,111 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40409/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726145821, journal: Splitting hdfs://localhost:40409/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726145821, size=0 (0bytes) at 1732726146092Finishing writing output for hdfs://localhost:40409/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726145821 so closing down at 1732726150100 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/recovered.edits/0000000000000000003-wal.1732726145821.temp at 1732726150102 (+2 ms)3 split writer threads finished at 1732726150103 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/recovered.edits/0000000000000000003-wal.1732726145821.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732726150109 (+6 ms)Rename recovered edits hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/recovered.edits/0000000000000000003-wal.1732726145821.temp to hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/recovered.edits/0000000000000000008 at 1732726150111 (+2 ms)Processed 5 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:40409/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726145821, size=0, length=0, corrupted=false, cancelled=false at 1732726150111 2024-11-27T16:49:10,113 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40409/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726145821 to hdfs://localhost:40409/hbase/oldWALs/wal.1732726145821 2024-11-27T16:49:10,113 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/recovered.edits/0000000000000000008 2024-11-27T16:49:10,114 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:49:10,115 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:49:10,129 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726150116, exclude list is [], retry=0 2024-11-27T16:49:10,131 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:49:10,131 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:49:10,132 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:49:10,133 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726150116 2024-11-27T16:49:10,134 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:39655:39655)] 2024-11-27T16:49:10,134 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a26e33e0aaac41ddc6329f0f1cbe48d7, NAME => 'testCompactedBulkLoadedFiles,,1732726145754.a26e33e0aaac41ddc6329f0f1cbe48d7.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:49:10,134 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732726145754.a26e33e0aaac41ddc6329f0f1cbe48d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:49:10,134 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:10,134 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:10,136 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:10,137 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a26e33e0aaac41ddc6329f0f1cbe48d7 columnFamilyName a 2024-11-27T16:49:10,137 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:10,143 DEBUG [StoreFileOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 01e469d7302745c0b12122d7f5b49e4b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T16:49:10,144 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/01e469d7302745c0b12122d7f5b49e4b_SeqId_4_ 2024-11-27T16:49:10,147 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/213df5ba45fa45cbbd4bb108a500231c 2024-11-27T16:49:10,152 DEBUG [StoreFileOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 35fce9a667db4b7f9affad437508c9bd_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T16:49:10,152 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/35fce9a667db4b7f9affad437508c9bd_SeqId_4_ 2024-11-27T16:49:10,155 DEBUG [StoreFileOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 62b9c6b0e7b74a1b9da8daa5190a0f14_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T16:49:10,156 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/62b9c6b0e7b74a1b9da8daa5190a0f14_SeqId_4_ 2024-11-27T16:49:10,159 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/be8f070fefbb40db8e98b2870006ba3c 2024-11-27T16:49:10,159 WARN [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/01e469d7302745c0b12122d7f5b49e4b_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@46ffc559 2024-11-27T16:49:10,159 WARN [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/35fce9a667db4b7f9affad437508c9bd_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@46ffc559 2024-11-27T16:49:10,159 WARN [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/62b9c6b0e7b74a1b9da8daa5190a0f14_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@46ffc559 2024-11-27T16:49:10,159 WARN [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/be8f070fefbb40db8e98b2870006ba3c from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@46ffc559 2024-11-27T16:49:10,159 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/01e469d7302745c0b12122d7f5b49e4b_SeqId_4_, hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/35fce9a667db4b7f9affad437508c9bd_SeqId_4_, hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/62b9c6b0e7b74a1b9da8daa5190a0f14_SeqId_4_, hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/be8f070fefbb40db8e98b2870006ba3c] to archive 2024-11-27T16:49:10,160 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T16:49:10,162 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/01e469d7302745c0b12122d7f5b49e4b_SeqId_4_ to hdfs://localhost:40409/hbase/archive/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/01e469d7302745c0b12122d7f5b49e4b_SeqId_4_ 2024-11-27T16:49:10,164 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/35fce9a667db4b7f9affad437508c9bd_SeqId_4_ to hdfs://localhost:40409/hbase/archive/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/35fce9a667db4b7f9affad437508c9bd_SeqId_4_ 2024-11-27T16:49:10,165 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/62b9c6b0e7b74a1b9da8daa5190a0f14_SeqId_4_ to hdfs://localhost:40409/hbase/archive/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/62b9c6b0e7b74a1b9da8daa5190a0f14_SeqId_4_ 2024-11-27T16:49:10,200 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/be8f070fefbb40db8e98b2870006ba3c to hdfs://localhost:40409/hbase/archive/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/a/be8f070fefbb40db8e98b2870006ba3c 2024-11-27T16:49:10,200 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.HStore(327): Store=a26e33e0aaac41ddc6329f0f1cbe48d7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:10,201 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:10,202 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a26e33e0aaac41ddc6329f0f1cbe48d7 columnFamilyName b 2024-11-27T16:49:10,202 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:10,203 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.HStore(327): Store=a26e33e0aaac41ddc6329f0f1cbe48d7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:10,203 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:10,204 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a26e33e0aaac41ddc6329f0f1cbe48d7 columnFamilyName c 2024-11-27T16:49:10,204 DEBUG [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:10,204 INFO [StoreOpener-a26e33e0aaac41ddc6329f0f1cbe48d7-1 {}] regionserver.HStore(327): Store=a26e33e0aaac41ddc6329f0f1cbe48d7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:10,204 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:10,205 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:10,207 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:10,207 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/recovered.edits/0000000000000000008 2024-11-27T16:49:10,209 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:49:10,212 DEBUG [Time-limited test {}] regionserver.HRegion(5836): a26e33e0aaac41ddc6329f0f1cbe48d7 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "a26e33e0aaac41ddc6329f0f1cbe48d7" family_name: "a" compaction_input: "be8f070fefbb40db8e98b2870006ba3c" compaction_input: "62b9c6b0e7b74a1b9da8daa5190a0f14_SeqId_4_" compaction_input: "01e469d7302745c0b12122d7f5b49e4b_SeqId_4_" compaction_input: "35fce9a667db4b7f9affad437508c9bd_SeqId_4_" compaction_output: "213df5ba45fa45cbbd4bb108a500231c" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732726145754.a26e33e0aaac41ddc6329f0f1cbe48d7." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-27T16:49:10,212 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-27T16:49:10,212 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/recovered.edits/0000000000000000008 2024-11-27T16:49:10,213 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/recovered.edits/0000000000000000008 2024-11-27T16:49:10,214 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:10,214 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:10,214 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T16:49:10,215 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T16:49:10,216 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a26e33e0aaac41ddc6329f0f1cbe48d7 2024-11-27T16:49:10,220 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testCompactedBulkLoadedFiles/a26e33e0aaac41ddc6329f0f1cbe48d7/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-27T16:49:10,221 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a26e33e0aaac41ddc6329f0f1cbe48d7; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61722538, jitterRate=-0.08026251196861267}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T16:49:10,221 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a26e33e0aaac41ddc6329f0f1cbe48d7: Writing region info on filesystem at 1732726150135Initializing all the Stores at 1732726150135Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726150135Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726150136 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726150136Cleaning up temporary data from old regions at 1732726150214 (+78 ms)Region opened successfully at 1732726150221 (+7 ms) 2024-11-27T16:49:10,224 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a26e33e0aaac41ddc6329f0f1cbe48d7, disabling compactions & flushes 2024-11-27T16:49:10,224 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732726145754.a26e33e0aaac41ddc6329f0f1cbe48d7. 2024-11-27T16:49:10,224 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732726145754.a26e33e0aaac41ddc6329f0f1cbe48d7. 2024-11-27T16:49:10,224 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732726145754.a26e33e0aaac41ddc6329f0f1cbe48d7. after waiting 0 ms 2024-11-27T16:49:10,224 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732726145754.a26e33e0aaac41ddc6329f0f1cbe48d7. 2024-11-27T16:49:10,225 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732726145754.a26e33e0aaac41ddc6329f0f1cbe48d7. 2024-11-27T16:49:10,225 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a26e33e0aaac41ddc6329f0f1cbe48d7: Waiting for close lock at 1732726150224Disabling compacts and flushes for region at 1732726150224Disabling writes for close at 1732726150224Writing region close event to WAL at 1732726150225 (+1 ms)Closed at 1732726150225 2024-11-27T16:49:10,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741924_1103 (size=95) 2024-11-27T16:49:10,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741924_1103 (size=95) 2024-11-27T16:49:10,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741924_1103 (size=95) 2024-11-27T16:49:10,234 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T16:49:10,234 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732726150116) 2024-11-27T16:49:10,254 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=440 (was 425) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_482808082_22 at /127.0.0.1:43878 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1112330208) connection to localhost/127.0.0.1:40409 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_482808082_22 at /127.0.0.1:49324 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_482808082_22 at /127.0.0.1:45142 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:40409 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1344 (was 1262) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=235 (was 247), ProcessCount=11 (was 11), AvailableMemoryMB=6317 (was 6374) 2024-11-27T16:49:10,254 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-11-27T16:49:10,271 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=440, OpenFileDescriptor=1344, MaxFileDescriptor=1048576, SystemLoadAverage=235, ProcessCount=11, AvailableMemoryMB=6315 2024-11-27T16:49:10,271 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-11-27T16:49:10,295 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:49:10,298 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T16:49:10,298 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T16:49:10,301 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-25675165, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/hregion-25675165, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:49:10,315 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-25675165/hregion-25675165.1732726150301, exclude list is [], retry=0 2024-11-27T16:49:10,318 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:49:10,319 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:49:10,319 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:49:10,322 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-25675165/hregion-25675165.1732726150301 2024-11-27T16:49:10,322 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:39655:39655),(127.0.0.1/127.0.0.1:42883:42883)] 2024-11-27T16:49:10,323 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 24b782ae0f288e33e8949d6735433f38, NAME => 'testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40409/hbase 2024-11-27T16:49:10,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741926_1105 (size=67) 2024-11-27T16:49:10,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741926_1105 (size=67) 2024-11-27T16:49:10,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741926_1105 (size=67) 2024-11-27T16:49:10,335 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:49:10,336 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,337 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24b782ae0f288e33e8949d6735433f38 columnFamilyName a 2024-11-27T16:49:10,337 DEBUG [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:10,338 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(327): Store=24b782ae0f288e33e8949d6735433f38/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:10,338 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,339 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24b782ae0f288e33e8949d6735433f38 columnFamilyName b 2024-11-27T16:49:10,339 DEBUG [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:10,340 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(327): Store=24b782ae0f288e33e8949d6735433f38/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:10,340 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,341 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24b782ae0f288e33e8949d6735433f38 columnFamilyName c 2024-11-27T16:49:10,341 DEBUG [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:10,342 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(327): Store=24b782ae0f288e33e8949d6735433f38/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:10,342 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,343 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,343 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,344 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,344 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,345 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T16:49:10,346 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,348 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T16:49:10,348 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 24b782ae0f288e33e8949d6735433f38; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61684658, jitterRate=-0.08082696795463562}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T16:49:10,349 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 24b782ae0f288e33e8949d6735433f38: Writing region info on filesystem at 1732726150335Initializing all the Stores at 1732726150335Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726150336 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726150336Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726150336Cleaning up temporary data from old regions at 1732726150344 (+8 ms)Region opened successfully at 1732726150349 (+5 ms) 2024-11-27T16:49:10,349 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 24b782ae0f288e33e8949d6735433f38, disabling compactions & flushes 2024-11-27T16:49:10,349 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38. 2024-11-27T16:49:10,349 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38. 2024-11-27T16:49:10,349 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38. after waiting 0 ms 2024-11-27T16:49:10,349 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38. 2024-11-27T16:49:10,349 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38. 2024-11-27T16:49:10,349 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 24b782ae0f288e33e8949d6735433f38: Waiting for close lock at 1732726150349Disabling compacts and flushes for region at 1732726150349Disabling writes for close at 1732726150349Writing region close event to WAL at 1732726150349Closed at 1732726150349 2024-11-27T16:49:10,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741925_1104 (size=95) 2024-11-27T16:49:10,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741925_1104 (size=95) 2024-11-27T16:49:10,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741925_1104 (size=95) 2024-11-27T16:49:10,355 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T16:49:10,355 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-25675165:(num 1732726150301) 2024-11-27T16:49:10,356 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:49:10,357 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:49:10,371 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150358, exclude list is [], retry=0 2024-11-27T16:49:10,374 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:49:10,375 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:49:10,375 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:49:10,377 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150358 2024-11-27T16:49:10,377 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:39655:39655)] 2024-11-27T16:49:10,377 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 24b782ae0f288e33e8949d6735433f38, NAME => 'testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:49:10,378 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:49:10,378 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,378 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,379 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,380 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24b782ae0f288e33e8949d6735433f38 columnFamilyName a 2024-11-27T16:49:10,380 DEBUG [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:10,381 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(327): Store=24b782ae0f288e33e8949d6735433f38/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:10,381 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,381 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24b782ae0f288e33e8949d6735433f38 columnFamilyName b 2024-11-27T16:49:10,382 DEBUG [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:10,382 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(327): Store=24b782ae0f288e33e8949d6735433f38/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:10,382 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,383 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24b782ae0f288e33e8949d6735433f38 columnFamilyName c 2024-11-27T16:49:10,383 DEBUG [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:10,383 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(327): Store=24b782ae0f288e33e8949d6735433f38/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:10,383 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,384 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,385 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,386 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,386 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,387 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T16:49:10,388 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,389 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 24b782ae0f288e33e8949d6735433f38; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59502526, jitterRate=-0.1133432686328888}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T16:49:10,389 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 24b782ae0f288e33e8949d6735433f38: Writing region info on filesystem at 1732726150378Initializing all the Stores at 1732726150379 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726150379Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726150379Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726150379Cleaning up temporary data from old regions at 1732726150386 (+7 ms)Region opened successfully at 1732726150389 (+3 ms) 2024-11-27T16:49:10,398 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 24b782ae0f288e33e8949d6735433f38 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-27T16:49:10,415 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/.tmp/a/668ae0d27e144bc5a3b723e72a0e200c is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732726150389/Put/seqid=0 2024-11-27T16:49:10,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741928_1107 (size=5958) 2024-11-27T16:49:10,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741928_1107 (size=5958) 2024-11-27T16:49:10,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741928_1107 (size=5958) 2024-11-27T16:49:10,422 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/.tmp/a/668ae0d27e144bc5a3b723e72a0e200c 2024-11-27T16:49:10,428 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/.tmp/a/668ae0d27e144bc5a3b723e72a0e200c as hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/a/668ae0d27e144bc5a3b723e72a0e200c 2024-11-27T16:49:10,434 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/a/668ae0d27e144bc5a3b723e72a0e200c, entries=10, sequenceid=13, filesize=5.8 K 2024-11-27T16:49:10,435 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 24b782ae0f288e33e8949d6735433f38 in 37ms, sequenceid=13, compaction requested=false 2024-11-27T16:49:10,435 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 24b782ae0f288e33e8949d6735433f38: 2024-11-27T16:49:10,453 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 24b782ae0f288e33e8949d6735433f38, disabling compactions & flushes 2024-11-27T16:49:10,453 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38. 2024-11-27T16:49:10,453 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38. 2024-11-27T16:49:10,453 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38. after waiting 0 ms 2024-11-27T16:49:10,453 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38. 2024-11-27T16:49:10,455 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38. 2024-11-27T16:49:10,455 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38. 2024-11-27T16:49:10,455 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 24b782ae0f288e33e8949d6735433f38: Waiting for close lock at 1732726150453Disabling compacts and flushes for region at 1732726150453Disabling writes for close at 1732726150453Writing region close event to WAL at 1732726150454 (+1 ms)Closed at 1732726150455 (+1 ms) 2024-11-27T16:49:10,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741927_1106 (size=3345) 2024-11-27T16:49:10,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741927_1106 (size=3345) 2024-11-27T16:49:10,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741927_1106 (size=3345) 2024-11-27T16:49:10,475 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150358, size=3.3 K (3345bytes) 2024-11-27T16:49:10,475 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150358 2024-11-27T16:49:10,475 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150358 after 0ms 2024-11-27T16:49:10,478 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150358: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:49:10,478 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150358 took 3ms 2024-11-27T16:49:10,481 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150358 so closing down 2024-11-27T16:49:10,481 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T16:49:10,482 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732726150358.temp 2024-11-27T16:49:10,483 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000003-wal.1732726150358.temp 2024-11-27T16:49:10,484 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T16:49:10,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741929_1108 (size=2944) 2024-11-27T16:49:10,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741929_1108 (size=2944) 2024-11-27T16:49:10,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741929_1108 (size=2944) 2024-11-27T16:49:10,508 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000003-wal.1732726150358.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-27T16:49:10,510 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000003-wal.1732726150358.temp to hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000035 2024-11-27T16:49:10,510 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 32 ms; skipped=2; WAL=hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150358, size=3.3 K, length=3345, corrupted=false, cancelled=false 2024-11-27T16:49:10,510 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150358, journal: Splitting hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150358, size=3.3 K (3345bytes) at 1732726150475Finishing writing output for hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150358 so closing down at 1732726150481 (+6 ms)Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000003-wal.1732726150358.temp at 1732726150483 (+2 ms)3 split writer threads finished at 1732726150484 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000003-wal.1732726150358.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732726150508 (+24 ms)Rename recovered edits hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000003-wal.1732726150358.temp to hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000035 at 1732726150510 (+2 ms)Processed 32 edits across 1 Regions in 32 ms; skipped=2; WAL=hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150358, size=3.3 K, length=3345, corrupted=false, cancelled=false at 1732726150510 2024-11-27T16:49:10,512 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150358 to hdfs://localhost:40409/hbase/oldWALs/wal.1732726150358 2024-11-27T16:49:10,513 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000035 2024-11-27T16:49:10,513 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:49:10,515 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:49:10,529 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150515, exclude list is [], retry=0 2024-11-27T16:49:10,531 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:49:10,531 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:49:10,532 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:49:10,534 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150515 2024-11-27T16:49:10,535 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:39655:39655)] 2024-11-27T16:49:10,535 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 24b782ae0f288e33e8949d6735433f38, NAME => 'testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38.', STARTKEY => '', ENDKEY => ''} 2024-11-27T16:49:10,535 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:49:10,535 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,535 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,536 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,537 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24b782ae0f288e33e8949d6735433f38 columnFamilyName a 2024-11-27T16:49:10,537 DEBUG [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:10,543 DEBUG [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/a/668ae0d27e144bc5a3b723e72a0e200c 2024-11-27T16:49:10,543 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(327): Store=24b782ae0f288e33e8949d6735433f38/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:10,543 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,544 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24b782ae0f288e33e8949d6735433f38 columnFamilyName b 2024-11-27T16:49:10,544 DEBUG [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:10,545 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(327): Store=24b782ae0f288e33e8949d6735433f38/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:10,545 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,545 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24b782ae0f288e33e8949d6735433f38 columnFamilyName c 2024-11-27T16:49:10,546 DEBUG [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:10,546 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(327): Store=24b782ae0f288e33e8949d6735433f38/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:10,546 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,547 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,548 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,549 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000035 2024-11-27T16:49:10,551 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:49:10,552 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000035 2024-11-27T16:49:10,552 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 24b782ae0f288e33e8949d6735433f38 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-27T16:49:10,574 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/.tmp/b/c90415d530a04ccca6dd8c73d36af108 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732726150435/Put/seqid=0 2024-11-27T16:49:10,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741931_1110 (size=5958) 2024-11-27T16:49:10,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741931_1110 (size=5958) 2024-11-27T16:49:10,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741931_1110 (size=5958) 2024-11-27T16:49:10,583 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/.tmp/b/c90415d530a04ccca6dd8c73d36af108 2024-11-27T16:49:10,603 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/.tmp/c/48f7c167baa94e2e839d1dc44a5b2b5e is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732726150442/Put/seqid=0 2024-11-27T16:49:10,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741932_1111 (size=5958) 2024-11-27T16:49:10,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741932_1111 (size=5958) 2024-11-27T16:49:10,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741932_1111 (size=5958) 2024-11-27T16:49:10,612 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/.tmp/c/48f7c167baa94e2e839d1dc44a5b2b5e 2024-11-27T16:49:10,618 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/.tmp/b/c90415d530a04ccca6dd8c73d36af108 as hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/b/c90415d530a04ccca6dd8c73d36af108 2024-11-27T16:49:10,625 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/b/c90415d530a04ccca6dd8c73d36af108, entries=10, sequenceid=35, filesize=5.8 K 2024-11-27T16:49:10,626 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/.tmp/c/48f7c167baa94e2e839d1dc44a5b2b5e as hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/c/48f7c167baa94e2e839d1dc44a5b2b5e 2024-11-27T16:49:10,632 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/c/48f7c167baa94e2e839d1dc44a5b2b5e, entries=10, sequenceid=35, filesize=5.8 K 2024-11-27T16:49:10,632 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 24b782ae0f288e33e8949d6735433f38 in 80ms, sequenceid=35, compaction requested=false; wal=null 2024-11-27T16:49:10,633 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000035 2024-11-27T16:49:10,634 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,634 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,635 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T16:49:10,636 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:10,639 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-27T16:49:10,640 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 24b782ae0f288e33e8949d6735433f38; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61654992, jitterRate=-0.0812690258026123}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T16:49:10,640 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 24b782ae0f288e33e8949d6735433f38: Writing region info on filesystem at 1732726150535Initializing all the Stores at 1732726150536 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726150536Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726150536Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726150536Obtaining lock to block concurrent updates at 1732726150552 (+16 ms)Preparing flush snapshotting stores in 24b782ae0f288e33e8949d6735433f38 at 1732726150552Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732726150553 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38. at 1732726150553Flushing 24b782ae0f288e33e8949d6735433f38/b: creating writer at 1732726150553Flushing 24b782ae0f288e33e8949d6735433f38/b: appending metadata at 1732726150573 (+20 ms)Flushing 24b782ae0f288e33e8949d6735433f38/b: closing flushed file at 1732726150573Flushing 24b782ae0f288e33e8949d6735433f38/c: creating writer at 1732726150588 (+15 ms)Flushing 24b782ae0f288e33e8949d6735433f38/c: appending metadata at 1732726150602 (+14 ms)Flushing 24b782ae0f288e33e8949d6735433f38/c: closing flushed file at 1732726150602Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36cb4bf6: reopening flushed file at 1732726150618 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a543f0c: reopening flushed file at 1732726150625 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 24b782ae0f288e33e8949d6735433f38 in 80ms, sequenceid=35, compaction requested=false; wal=null at 1732726150632 (+7 ms)Cleaning up temporary data from old regions at 1732726150634 (+2 ms)Region opened successfully at 1732726150640 (+6 ms) 2024-11-27T16:49:10,717 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150515, size=0 (0bytes) 2024-11-27T16:49:10,717 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150515 might be still open, length is 0 2024-11-27T16:49:10,717 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150515 2024-11-27T16:49:10,718 WARN [IPC Server handler 0 on default port 40409 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150515 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-27T16:49:10,718 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150515 after 1ms 2024-11-27T16:49:13,466 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:45224 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:37433:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45224 dst: /127.0.0.1:37433 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37433 remote=/127.0.0.1:45224]. Total timeout mills is 60000, 57208 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:49:13,466 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:43930 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:41739:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43930 dst: /127.0.0.1:41739 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:49:13,467 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-214764288_22 at /127.0.0.1:49386 [Receiving block BP-89784467-172.17.0.2-1732726112257:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:37511:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49386 dst: /127.0.0.1:37511 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:49:13,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741930_1112 (size=2936) 2024-11-27T16:49:13,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741930_1112 (size=2936) 2024-11-27T16:49:13,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741930_1112 (size=2936) 2024-11-27T16:49:14,719 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150515 after 4002ms 2024-11-27T16:49:14,722 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150515: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:49:14,722 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150515 took 4005ms 2024-11-27T16:49:14,724 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150515; continuing. 2024-11-27T16:49:14,724 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150515 so closing down 2024-11-27T16:49:14,724 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T16:49:14,726 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732726150515.temp 2024-11-27T16:49:14,727 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000037-wal.1732726150515.temp 2024-11-27T16:49:14,728 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T16:49:14,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741933_1113 (size=2944) 2024-11-27T16:49:14,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741933_1113 (size=2944) 2024-11-27T16:49:14,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741933_1113 (size=2944) 2024-11-27T16:49:14,741 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000037-wal.1732726150515.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-27T16:49:14,742 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000037-wal.1732726150515.temp to hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000066 2024-11-27T16:49:14,743 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150515, size=0, length=0, corrupted=false, cancelled=false 2024-11-27T16:49:14,743 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150515, journal: Splitting hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150515, size=0 (0bytes) at 1732726150717Finishing writing output for hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150515 so closing down at 1732726154724 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000037-wal.1732726150515.temp at 1732726154727 (+3 ms)3 split writer threads finished at 1732726154728 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000037-wal.1732726150515.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732726154741 (+13 ms)Rename recovered edits hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000037-wal.1732726150515.temp to hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000066 at 1732726154743 (+2 ms)Processed 30 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150515, size=0, length=0, corrupted=false, cancelled=false at 1732726154743 2024-11-27T16:49:14,744 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150515 to hdfs://localhost:40409/hbase/oldWALs/wal.1732726150515 2024-11-27T16:49:14,745 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000066 2024-11-27T16:49:14,745 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T16:49:14,747 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40409/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295, archiveDir=hdfs://localhost:40409/hbase/oldWALs, maxLogs=32 2024-11-27T16:49:14,760 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726154747, exclude list is [], retry=0 2024-11-27T16:49:14,763 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41739,DS-a471d440-70d4-449b-9f22-dafdb2116171,DISK] 2024-11-27T16:49:14,763 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37433,DS-1d9a0fcc-b23c-499c-b2e5-2ad4e1b073f7,DISK] 2024-11-27T16:49:14,763 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37511,DS-ed211189-b496-4df4-badd-0783b484a6d0,DISK] 2024-11-27T16:49:14,765 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726154747 2024-11-27T16:49:14,765 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33829:33829),(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:39655:39655)] 2024-11-27T16:49:14,766 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T16:49:14,767 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:14,768 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24b782ae0f288e33e8949d6735433f38 columnFamilyName a 2024-11-27T16:49:14,768 DEBUG [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:14,775 DEBUG [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/a/668ae0d27e144bc5a3b723e72a0e200c 2024-11-27T16:49:14,776 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(327): Store=24b782ae0f288e33e8949d6735433f38/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:14,776 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:14,777 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24b782ae0f288e33e8949d6735433f38 columnFamilyName b 2024-11-27T16:49:14,777 DEBUG [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:14,783 DEBUG [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/b/c90415d530a04ccca6dd8c73d36af108 2024-11-27T16:49:14,783 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(327): Store=24b782ae0f288e33e8949d6735433f38/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:14,783 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:14,784 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24b782ae0f288e33e8949d6735433f38 columnFamilyName c 2024-11-27T16:49:14,784 DEBUG [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T16:49:14,789 DEBUG [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/c/48f7c167baa94e2e839d1dc44a5b2b5e 2024-11-27T16:49:14,789 INFO [StoreOpener-24b782ae0f288e33e8949d6735433f38-1 {}] regionserver.HStore(327): Store=24b782ae0f288e33e8949d6735433f38/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T16:49:14,789 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:14,790 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:14,791 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:14,792 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000066 2024-11-27T16:49:14,793 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T16:49:14,798 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000066 2024-11-27T16:49:14,798 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 24b782ae0f288e33e8949d6735433f38 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-27T16:49:14,813 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/.tmp/a/372b6686d5a74257b0c294ea14fae629 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732726150648/Put/seqid=0 2024-11-27T16:49:14,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741935_1115 (size=5958) 2024-11-27T16:49:14,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741935_1115 (size=5958) 2024-11-27T16:49:14,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741935_1115 (size=5958) 2024-11-27T16:49:14,820 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/.tmp/a/372b6686d5a74257b0c294ea14fae629 2024-11-27T16:49:14,839 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/.tmp/b/c3f2668f9f3540478f2a671282f921f6 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732726150658/Put/seqid=0 2024-11-27T16:49:14,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741936_1116 (size=5958) 2024-11-27T16:49:14,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741936_1116 (size=5958) 2024-11-27T16:49:14,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741936_1116 (size=5958) 2024-11-27T16:49:14,846 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/.tmp/b/c3f2668f9f3540478f2a671282f921f6 2024-11-27T16:49:14,871 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/.tmp/c/5a2efdb180a54a50b21a47f21b40d429 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732726150665/Put/seqid=0 2024-11-27T16:49:14,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741937_1117 (size=5958) 2024-11-27T16:49:14,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741937_1117 (size=5958) 2024-11-27T16:49:14,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741937_1117 (size=5958) 2024-11-27T16:49:14,882 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/.tmp/c/5a2efdb180a54a50b21a47f21b40d429 2024-11-27T16:49:14,887 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/.tmp/a/372b6686d5a74257b0c294ea14fae629 as hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/a/372b6686d5a74257b0c294ea14fae629 2024-11-27T16:49:14,891 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/a/372b6686d5a74257b0c294ea14fae629, entries=10, sequenceid=66, filesize=5.8 K 2024-11-27T16:49:14,892 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/.tmp/b/c3f2668f9f3540478f2a671282f921f6 as hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/b/c3f2668f9f3540478f2a671282f921f6 2024-11-27T16:49:14,896 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/b/c3f2668f9f3540478f2a671282f921f6, entries=10, sequenceid=66, filesize=5.8 K 2024-11-27T16:49:14,897 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/.tmp/c/5a2efdb180a54a50b21a47f21b40d429 as hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/c/5a2efdb180a54a50b21a47f21b40d429 2024-11-27T16:49:14,901 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/c/5a2efdb180a54a50b21a47f21b40d429, entries=10, sequenceid=66, filesize=5.8 K 2024-11-27T16:49:14,901 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 24b782ae0f288e33e8949d6735433f38 in 103ms, sequenceid=66, compaction requested=false; wal=null 2024-11-27T16:49:14,902 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/0000000000000000066 2024-11-27T16:49:14,903 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:14,903 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:14,903 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T16:49:14,905 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 24b782ae0f288e33e8949d6735433f38 2024-11-27T16:49:14,907 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/hbase/data/default/testReplayEditsWrittenViaHRegion/24b782ae0f288e33e8949d6735433f38/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-27T16:49:14,908 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 24b782ae0f288e33e8949d6735433f38; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71067063, jitterRate=0.0589817613363266}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T16:49:14,908 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 24b782ae0f288e33e8949d6735433f38: Writing region info on filesystem at 1732726154766Initializing all the Stores at 1732726154766Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726154767 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726154767Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732726154767Obtaining lock to block concurrent updates at 1732726154799 (+32 ms)Preparing flush snapshotting stores in 24b782ae0f288e33e8949d6735433f38 at 1732726154799Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732726154799Flushing stores of testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38. at 1732726154799Flushing 24b782ae0f288e33e8949d6735433f38/a: creating writer at 1732726154799Flushing 24b782ae0f288e33e8949d6735433f38/a: appending metadata at 1732726154813 (+14 ms)Flushing 24b782ae0f288e33e8949d6735433f38/a: closing flushed file at 1732726154813Flushing 24b782ae0f288e33e8949d6735433f38/b: creating writer at 1732726154825 (+12 ms)Flushing 24b782ae0f288e33e8949d6735433f38/b: appending metadata at 1732726154838 (+13 ms)Flushing 24b782ae0f288e33e8949d6735433f38/b: closing flushed file at 1732726154838Flushing 24b782ae0f288e33e8949d6735433f38/c: creating writer at 1732726154851 (+13 ms)Flushing 24b782ae0f288e33e8949d6735433f38/c: appending metadata at 1732726154871 (+20 ms)Flushing 24b782ae0f288e33e8949d6735433f38/c: closing flushed file at 1732726154871Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25417871: reopening flushed file at 1732726154886 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c5e166b: reopening flushed file at 1732726154891 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3de92385: reopening flushed file at 1732726154896 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 24b782ae0f288e33e8949d6735433f38 in 103ms, sequenceid=66, compaction requested=false; wal=null at 1732726154901 (+5 ms)Cleaning up temporary data from old regions at 1732726154903 (+2 ms)Region opened successfully at 1732726154908 (+5 ms) 2024-11-27T16:49:14,923 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 24b782ae0f288e33e8949d6735433f38, disabling compactions & flushes 2024-11-27T16:49:14,923 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38. 2024-11-27T16:49:14,923 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38. 2024-11-27T16:49:14,923 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38. after waiting 0 ms 2024-11-27T16:49:14,923 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38. 2024-11-27T16:49:14,924 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732726150296.24b782ae0f288e33e8949d6735433f38. 2024-11-27T16:49:14,924 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 24b782ae0f288e33e8949d6735433f38: Waiting for close lock at 1732726154923Disabling compacts and flushes for region at 1732726154923Disabling writes for close at 1732726154923Writing region close event to WAL at 1732726154924 (+1 ms)Closed at 1732726154924 2024-11-27T16:49:14,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741934_1114 (size=95) 2024-11-27T16:49:14,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741934_1114 (size=95) 2024-11-27T16:49:14,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741934_1114 (size=95) 2024-11-27T16:49:14,929 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T16:49:14,929 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732726154747) 2024-11-27T16:49:14,946 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=445 (was 440) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:40409 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_933259378_22 at /127.0.0.1:41312 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1112330208) connection to localhost/127.0.0.1:40409 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_933259378_22 at /127.0.0.1:47674 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_933259378_22 at /127.0.0.1:58680 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1410 (was 1344) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=232 (was 235), ProcessCount=11 (was 11), AvailableMemoryMB=6303 (was 6315) 2024-11-27T16:49:14,946 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1410 is superior to 1024 2024-11-27T16:49:14,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T16:49:14,946 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T16:49:14,946 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T16:49:14,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T16:49:14,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T16:49:14,947 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T16:49:14,947 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T16:49:14,947 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1116026814, stopped=false 2024-11-27T16:49:14,947 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=37bfedfd9766,35959,1732726115618 2024-11-27T16:49:14,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T16:49:14,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T16:49:14,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:49:14,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T16:49:14,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:49:14,950 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T16:49:14,950 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T16:49:14,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:49:14,951 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T16:49:14,952 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T16:49:14,952 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T16:49:14,952 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T16:49:14,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T16:49:14,952 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '37bfedfd9766,40989,1732726116343' ***** 2024-11-27T16:49:14,952 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T16:49:14,952 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '37bfedfd9766,36103,1732726116491' ***** 2024-11-27T16:49:14,952 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T16:49:14,953 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T16:49:14,953 INFO [RS:2;37bfedfd9766:36103 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T16:49:14,953 INFO [RS:0;37bfedfd9766:40989 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T16:49:14,953 INFO [RS:2;37bfedfd9766:36103 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T16:49:14,953 INFO [RS:0;37bfedfd9766:40989 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T16:49:14,953 INFO [RS:2;37bfedfd9766:36103 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T16:49:14,953 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer(3091): Received CLOSE for e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:49:14,953 INFO [RS:2;37bfedfd9766:36103 {}] regionserver.HRegionServer(959): stopping server 37bfedfd9766,36103,1732726116491 2024-11-27T16:49:14,953 INFO [RS:2;37bfedfd9766:36103 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T16:49:14,953 INFO [RS:2;37bfedfd9766:36103 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;37bfedfd9766:36103. 2024-11-27T16:49:14,953 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T16:49:14,953 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer(959): stopping server 37bfedfd9766,40989,1732726116343 2024-11-27T16:49:14,953 DEBUG [RS:2;37bfedfd9766:36103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T16:49:14,953 INFO [RS:0;37bfedfd9766:40989 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T16:49:14,953 DEBUG [RS:2;37bfedfd9766:36103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T16:49:14,953 INFO [RS:0;37bfedfd9766:40989 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;37bfedfd9766:40989. 2024-11-27T16:49:14,953 INFO [RS:2;37bfedfd9766:36103 {}] regionserver.HRegionServer(976): stopping server 37bfedfd9766,36103,1732726116491; all regions closed. 2024-11-27T16:49:14,953 DEBUG [RS:0;37bfedfd9766:40989 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T16:49:14,953 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e7d41a86458ce2565a8f7b7bb1771881, disabling compactions & flushes 2024-11-27T16:49:14,953 DEBUG [RS:0;37bfedfd9766:40989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T16:49:14,953 INFO [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:49:14,953 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:49:14,953 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T16:49:14,954 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. after waiting 0 ms 2024-11-27T16:49:14,954 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T16:49:14,954 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:49:14,954 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T16:49:14,954 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T16:49:14,954 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T16:49:14,954 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-27T16:49:14,954 DEBUG [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, e7d41a86458ce2565a8f7b7bb1771881=testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881.} 2024-11-27T16:49:14,954 DEBUG [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, e7d41a86458ce2565a8f7b7bb1771881 2024-11-27T16:49:14,955 DEBUG [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T16:49:14,955 INFO [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T16:49:14,955 DEBUG [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T16:49:14,955 DEBUG [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T16:49:14,955 DEBUG [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T16:49:14,955 INFO [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-27T16:49:14,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741835_1011 (size=95) 2024-11-27T16:49:14,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741835_1011 (size=95) 2024-11-27T16:49:14,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741835_1011 (size=95) 2024-11-27T16:49:14,960 DEBUG [RS:2;37bfedfd9766:36103 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/oldWALs 2024-11-27T16:49:14,960 INFO [RS:2;37bfedfd9766:36103 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 37bfedfd9766%2C36103%2C1732726116491:(num 1732726117955) 2024-11-27T16:49:14,960 DEBUG [RS:2;37bfedfd9766:36103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T16:49:14,960 INFO [RS:2;37bfedfd9766:36103 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T16:49:14,960 INFO [RS:2;37bfedfd9766:36103 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T16:49:14,960 INFO [RS:2;37bfedfd9766:36103 {}] hbase.ChoreService(370): Chore service for: regionserver/37bfedfd9766:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-27T16:49:14,960 INFO [RS:2;37bfedfd9766:36103 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T16:49:14,960 INFO [RS:2;37bfedfd9766:36103 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T16:49:14,960 INFO [RS:2;37bfedfd9766:36103 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T16:49:14,960 INFO [regionserver/37bfedfd9766:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T16:49:14,960 INFO [RS:2;37bfedfd9766:36103 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T16:49:14,960 INFO [RS:2;37bfedfd9766:36103 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36103 2024-11-27T16:49:14,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T16:49:14,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/37bfedfd9766,36103,1732726116491 2024-11-27T16:49:14,962 INFO [RS:2;37bfedfd9766:36103 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T16:49:14,964 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [37bfedfd9766,36103,1732726116491] 2024-11-27T16:49:14,966 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/37bfedfd9766,36103,1732726116491 already deleted, retry=false 2024-11-27T16:49:14,966 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 37bfedfd9766,36103,1732726116491 expired; onlineServers=1 2024-11-27T16:49:14,969 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e7d41a86458ce2565a8f7b7bb1771881/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-27T16:49:14,969 INFO [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:49:14,969 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e7d41a86458ce2565a8f7b7bb1771881: Waiting for close lock at 1732726154953Running coprocessor pre-close hooks at 1732726154953Disabling compacts and flushes for region at 1732726154953Disabling writes for close at 1732726154954 (+1 ms)Writing region close event to WAL at 1732726154964 (+10 ms)Running coprocessor post-close hooks at 1732726154969 (+5 ms)Closed at 1732726154969 2024-11-27T16:49:14,969 DEBUG [RS_CLOSE_REGION-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881. 2024-11-27T16:49:14,983 DEBUG [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740/.tmp/info/c0d58b6c966c45359e4f88163571a439 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732726131082.e7d41a86458ce2565a8f7b7bb1771881./info:regioninfo/1732726134687/Put/seqid=0 2024-11-27T16:49:14,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741938_1118 (size=8243) 2024-11-27T16:49:14,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741938_1118 (size=8243) 2024-11-27T16:49:14,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741938_1118 (size=8243) 2024-11-27T16:49:14,994 INFO [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740/.tmp/info/c0d58b6c966c45359e4f88163571a439 2024-11-27T16:49:15,015 DEBUG [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740/.tmp/ns/9af0282b85b544929f67551ea93f0775 is 43, key is default/ns:d/1732726118574/Put/seqid=0 2024-11-27T16:49:15,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741939_1119 (size=5153) 2024-11-27T16:49:15,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741939_1119 (size=5153) 2024-11-27T16:49:15,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741939_1119 (size=5153) 2024-11-27T16:49:15,024 INFO [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740/.tmp/ns/9af0282b85b544929f67551ea93f0775 2024-11-27T16:49:15,046 INFO [regionserver/37bfedfd9766:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T16:49:15,046 INFO [regionserver/37bfedfd9766:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T16:49:15,052 DEBUG [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740/.tmp/table/aee041e845d840a7aacec97e960ab680 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732726131500/Put/seqid=0 2024-11-27T16:49:15,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741940_1120 (size=5431) 2024-11-27T16:49:15,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741940_1120 (size=5431) 2024-11-27T16:49:15,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741940_1120 (size=5431) 2024-11-27T16:49:15,059 INFO [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740/.tmp/table/aee041e845d840a7aacec97e960ab680 2024-11-27T16:49:15,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T16:49:15,064 INFO [RS:2;37bfedfd9766:36103 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T16:49:15,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36103-0x10039e202d30003, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T16:49:15,064 INFO [RS:2;37bfedfd9766:36103 {}] regionserver.HRegionServer(1031): Exiting; stopping=37bfedfd9766,36103,1732726116491; zookeeper connection closed. 2024-11-27T16:49:15,065 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@63983270 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@63983270 2024-11-27T16:49:15,066 DEBUG [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740/.tmp/info/c0d58b6c966c45359e4f88163571a439 as hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740/info/c0d58b6c966c45359e4f88163571a439 2024-11-27T16:49:15,072 INFO [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740/info/c0d58b6c966c45359e4f88163571a439, entries=18, sequenceid=21, filesize=8.0 K 2024-11-27T16:49:15,074 DEBUG [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740/.tmp/ns/9af0282b85b544929f67551ea93f0775 as hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740/ns/9af0282b85b544929f67551ea93f0775 2024-11-27T16:49:15,079 INFO [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740/ns/9af0282b85b544929f67551ea93f0775, entries=2, sequenceid=21, filesize=5.0 K 2024-11-27T16:49:15,080 DEBUG [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740/.tmp/table/aee041e845d840a7aacec97e960ab680 as hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740/table/aee041e845d840a7aacec97e960ab680 2024-11-27T16:49:15,085 INFO [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740/table/aee041e845d840a7aacec97e960ab680, entries=2, sequenceid=21, filesize=5.3 K 2024-11-27T16:49:15,086 INFO [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 131ms, sequenceid=21, compaction requested=false 2024-11-27T16:49:15,091 DEBUG [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-27T16:49:15,091 DEBUG [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T16:49:15,091 INFO [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T16:49:15,091 DEBUG [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732726154954Running coprocessor pre-close hooks at 1732726154954Disabling compacts and flushes for region at 1732726154954Disabling writes for close at 1732726154955 (+1 ms)Obtaining lock to block concurrent updates at 1732726154955Preparing flush snapshotting stores in 1588230740 at 1732726154955Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1732726154956 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732726154957 (+1 ms)Flushing 1588230740/info: creating writer at 1732726154957Flushing 1588230740/info: appending metadata at 1732726154982 (+25 ms)Flushing 1588230740/info: closing flushed file at 1732726154982Flushing 1588230740/ns: creating writer at 1732726155000 (+18 ms)Flushing 1588230740/ns: appending metadata at 1732726155014 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732726155014Flushing 1588230740/table: creating writer at 1732726155030 (+16 ms)Flushing 1588230740/table: appending metadata at 1732726155051 (+21 ms)Flushing 1588230740/table: closing flushed file at 1732726155051Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bdd1410: reopening flushed file at 1732726155065 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@593a8d8b: reopening flushed file at 1732726155073 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b2eae9b: reopening flushed file at 1732726155079 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 131ms, sequenceid=21, compaction requested=false at 1732726155086 (+7 ms)Writing region close event to WAL at 1732726155088 (+2 ms)Running coprocessor post-close hooks at 1732726155091 (+3 ms)Closed at 1732726155091 2024-11-27T16:49:15,092 DEBUG [RS_CLOSE_META-regionserver/37bfedfd9766:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T16:49:15,155 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer(976): stopping server 37bfedfd9766,40989,1732726116343; all regions closed. 2024-11-27T16:49:15,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741836_1012 (size=4717) 2024-11-27T16:49:15,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741836_1012 (size=4717) 2024-11-27T16:49:15,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741836_1012 (size=4717) 2024-11-27T16:49:15,160 DEBUG [RS:0;37bfedfd9766:40989 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/oldWALs 2024-11-27T16:49:15,160 INFO [RS:0;37bfedfd9766:40989 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 37bfedfd9766%2C40989%2C1732726116343.meta:.meta(num 1732726118359) 2024-11-27T16:49:15,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741833_1009 (size=2174) 2024-11-27T16:49:15,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741833_1009 (size=2174) 2024-11-27T16:49:15,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741833_1009 (size=2174) 2024-11-27T16:49:15,165 DEBUG [RS:0;37bfedfd9766:40989 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/oldWALs 2024-11-27T16:49:15,165 INFO [RS:0;37bfedfd9766:40989 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 37bfedfd9766%2C40989%2C1732726116343:(num 1732726117955) 2024-11-27T16:49:15,165 DEBUG [RS:0;37bfedfd9766:40989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T16:49:15,165 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T16:49:15,165 INFO [RS:0;37bfedfd9766:40989 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T16:49:15,165 INFO [RS:0;37bfedfd9766:40989 {}] hbase.ChoreService(370): Chore service for: regionserver/37bfedfd9766:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T16:49:15,165 INFO [RS:0;37bfedfd9766:40989 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T16:49:15,165 INFO [regionserver/37bfedfd9766:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T16:49:15,165 INFO [RS:0;37bfedfd9766:40989 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40989 2024-11-27T16:49:15,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T16:49:15,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/37bfedfd9766,40989,1732726116343 2024-11-27T16:49:15,168 INFO [RS:0;37bfedfd9766:40989 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T16:49:15,169 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [37bfedfd9766,40989,1732726116343] 2024-11-27T16:49:15,170 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/37bfedfd9766,40989,1732726116343 already deleted, retry=false 2024-11-27T16:49:15,170 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 37bfedfd9766,40989,1732726116343 expired; onlineServers=0 2024-11-27T16:49:15,170 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '37bfedfd9766,35959,1732726115618' ***** 2024-11-27T16:49:15,170 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T16:49:15,171 INFO [M:0;37bfedfd9766:35959 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T16:49:15,171 INFO [M:0;37bfedfd9766:35959 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T16:49:15,171 DEBUG [M:0;37bfedfd9766:35959 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T16:49:15,171 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T16:49:15,171 DEBUG [M:0;37bfedfd9766:35959 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T16:49:15,171 DEBUG [master/37bfedfd9766:0:becomeActiveMaster-HFileCleaner.large.0-1732726117590 {}] cleaner.HFileCleaner(306): Exit Thread[master/37bfedfd9766:0:becomeActiveMaster-HFileCleaner.large.0-1732726117590,5,FailOnTimeoutGroup] 2024-11-27T16:49:15,171 DEBUG [master/37bfedfd9766:0:becomeActiveMaster-HFileCleaner.small.0-1732726117591 {}] cleaner.HFileCleaner(306): Exit Thread[master/37bfedfd9766:0:becomeActiveMaster-HFileCleaner.small.0-1732726117591,5,FailOnTimeoutGroup] 2024-11-27T16:49:15,171 INFO [M:0;37bfedfd9766:35959 {}] hbase.ChoreService(370): Chore service for: master/37bfedfd9766:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T16:49:15,171 INFO [M:0;37bfedfd9766:35959 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T16:49:15,171 DEBUG [M:0;37bfedfd9766:35959 {}] master.HMaster(1795): Stopping service threads 2024-11-27T16:49:15,171 INFO [M:0;37bfedfd9766:35959 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T16:49:15,172 INFO [M:0;37bfedfd9766:35959 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T16:49:15,172 INFO [M:0;37bfedfd9766:35959 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T16:49:15,172 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T16:49:15,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T16:49:15,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T16:49:15,173 DEBUG [M:0;37bfedfd9766:35959 {}] zookeeper.ZKUtil(347): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-27T16:49:15,173 WARN [M:0;37bfedfd9766:35959 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T16:49:15,174 INFO [M:0;37bfedfd9766:35959 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/.lastflushedseqids 2024-11-27T16:49:15,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741941_1121 (size=138) 2024-11-27T16:49:15,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741941_1121 (size=138) 2024-11-27T16:49:15,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741941_1121 (size=138) 2024-11-27T16:49:15,187 INFO [M:0;37bfedfd9766:35959 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T16:49:15,187 INFO [M:0;37bfedfd9766:35959 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T16:49:15,187 DEBUG [M:0;37bfedfd9766:35959 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T16:49:15,187 INFO [M:0;37bfedfd9766:35959 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T16:49:15,187 DEBUG [M:0;37bfedfd9766:35959 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T16:49:15,187 DEBUG [M:0;37bfedfd9766:35959 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T16:49:15,187 DEBUG [M:0;37bfedfd9766:35959 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T16:49:15,187 INFO [M:0;37bfedfd9766:35959 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-11-27T16:49:15,204 DEBUG [M:0;37bfedfd9766:35959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/91bb2218b66f4bc1b73c012021a34b5b is 82, key is hbase:meta,,1/info:regioninfo/1732726118459/Put/seqid=0 2024-11-27T16:49:15,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741942_1122 (size=5672) 2024-11-27T16:49:15,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741942_1122 (size=5672) 2024-11-27T16:49:15,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741942_1122 (size=5672) 2024-11-27T16:49:15,211 INFO [M:0;37bfedfd9766:35959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/91bb2218b66f4bc1b73c012021a34b5b 2024-11-27T16:49:15,231 DEBUG [M:0;37bfedfd9766:35959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fbf24a19393e4ec982cef28c5dd00880 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732726131511/Put/seqid=0 2024-11-27T16:49:15,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741943_1123 (size=7755) 2024-11-27T16:49:15,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741943_1123 (size=7755) 2024-11-27T16:49:15,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741943_1123 (size=7755) 2024-11-27T16:49:15,241 INFO [M:0;37bfedfd9766:35959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fbf24a19393e4ec982cef28c5dd00880 2024-11-27T16:49:15,245 INFO [M:0;37bfedfd9766:35959 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fbf24a19393e4ec982cef28c5dd00880 2024-11-27T16:49:15,261 DEBUG [M:0;37bfedfd9766:35959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2d01c4d8b30744c98014fcf02ba797ea is 69, key is 37bfedfd9766,36103,1732726116491/rs:state/1732726117711/Put/seqid=0 2024-11-27T16:49:15,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T16:49:15,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10039e202d30001, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T16:49:15,269 INFO [RS:0;37bfedfd9766:40989 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T16:49:15,269 INFO [RS:0;37bfedfd9766:40989 {}] regionserver.HRegionServer(1031): Exiting; stopping=37bfedfd9766,40989,1732726116343; zookeeper connection closed. 2024-11-27T16:49:15,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741944_1124 (size=5445) 2024-11-27T16:49:15,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741944_1124 (size=5445) 2024-11-27T16:49:15,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741944_1124 (size=5445) 2024-11-27T16:49:15,275 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@149ce6f3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@149ce6f3 2024-11-27T16:49:15,275 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-27T16:49:15,672 INFO [M:0;37bfedfd9766:35959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2d01c4d8b30744c98014fcf02ba797ea 2024-11-27T16:49:15,677 INFO [M:0;37bfedfd9766:35959 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2d01c4d8b30744c98014fcf02ba797ea 2024-11-27T16:49:15,678 DEBUG [M:0;37bfedfd9766:35959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/91bb2218b66f4bc1b73c012021a34b5b as hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/91bb2218b66f4bc1b73c012021a34b5b 2024-11-27T16:49:15,683 INFO [M:0;37bfedfd9766:35959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/91bb2218b66f4bc1b73c012021a34b5b, entries=8, sequenceid=168, filesize=5.5 K 2024-11-27T16:49:15,683 DEBUG [M:0;37bfedfd9766:35959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fbf24a19393e4ec982cef28c5dd00880 as hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fbf24a19393e4ec982cef28c5dd00880 2024-11-27T16:49:15,688 INFO [M:0;37bfedfd9766:35959 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fbf24a19393e4ec982cef28c5dd00880 2024-11-27T16:49:15,689 INFO [M:0;37bfedfd9766:35959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fbf24a19393e4ec982cef28c5dd00880, entries=17, sequenceid=168, filesize=7.6 K 2024-11-27T16:49:15,689 DEBUG [M:0;37bfedfd9766:35959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2d01c4d8b30744c98014fcf02ba797ea as hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2d01c4d8b30744c98014fcf02ba797ea 2024-11-27T16:49:15,694 INFO [M:0;37bfedfd9766:35959 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2d01c4d8b30744c98014fcf02ba797ea 2024-11-27T16:49:15,694 INFO [M:0;37bfedfd9766:35959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40409/user/jenkins/test-data/62ceb850-7536-6d3d-3ae3-4219df58bcc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2d01c4d8b30744c98014fcf02ba797ea, entries=3, sequenceid=168, filesize=5.3 K 2024-11-27T16:49:15,695 INFO [M:0;37bfedfd9766:35959 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 508ms, sequenceid=168, compaction requested=false 2024-11-27T16:49:15,696 INFO [M:0;37bfedfd9766:35959 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T16:49:15,696 DEBUG [M:0;37bfedfd9766:35959 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732726155187Disabling compacts and flushes for region at 1732726155187Disabling writes for close at 1732726155187Obtaining lock to block concurrent updates at 1732726155187Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732726155187Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1732726155188 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732726155188Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732726155188Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732726155203 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732726155203Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732726155215 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732726155230 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732726155230Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732726155245 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732726155260 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732726155260Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@456be16e: reopening flushed file at 1732726155677 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c9082d9: reopening flushed file at 1732726155683 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25fa9d87: reopening flushed file at 1732726155689 (+6 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 508ms, sequenceid=168, compaction requested=false at 1732726155695 (+6 ms)Writing region close event to WAL at 1732726155696 (+1 ms)Closed at 1732726155696 2024-11-27T16:49:15,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37511 is added to blk_1073741830_1006 (size=56509) 2024-11-27T16:49:15,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37433 is added to blk_1073741830_1006 (size=56509) 2024-11-27T16:49:15,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41739 is added to blk_1073741830_1006 (size=56509) 2024-11-27T16:49:15,700 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T16:49:15,700 INFO [M:0;37bfedfd9766:35959 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T16:49:15,700 INFO [M:0;37bfedfd9766:35959 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35959 2024-11-27T16:49:15,700 INFO [M:0;37bfedfd9766:35959 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T16:49:15,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T16:49:15,802 INFO [M:0;37bfedfd9766:35959 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T16:49:15,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35959-0x10039e202d30000, quorum=127.0.0.1:60075, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T16:49:15,810 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061/wal.1732726135359 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061/wal.1732726135359 (inode 16655) Holder DFSClient_NONMAPREDUCE_-214764288_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732726135061/wal.1732726135359 (inode 16655) Holder DFSClient_NONMAPREDUCE_-214764288_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-27T16:49:15,811 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732726126649/wal.1732726126761 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:49:15,812 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732726150295/wal.1732726150515 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:49:15,814 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506/wal.1732726144742 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506/wal.1732726144742 (inode 16678) Holder DFSClient_NONMAPREDUCE_-214764288_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732726135506/wal.1732726144742 (inode 16678) Holder DFSClient_NONMAPREDUCE_-214764288_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-27T16:49:15,815 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732726119714/wal.1732726119787 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:49:15,817 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999/wal.1732726145585 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999/wal.1732726145585 (inode 16704) Holder DFSClient_NONMAPREDUCE_-214764288_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732726144999/wal.1732726145585 (inode 16704) Holder DFSClient_NONMAPREDUCE_-214764288_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-27T16:49:15,818 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732726145753/wal.1732726145821 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T16:49:15,820 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal.1732726119571 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal.1732726119571 (inode 16485) Holder DFSClient_NONMAPREDUCE_-214764288_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732726119375/wal.1732726119571 (inode 16485) Holder DFSClient_NONMAPREDUCE_-214764288_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-27T16:49:15,822 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal.1732726119128 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal.1732726119128 (inode 16462) Holder DFSClient_NONMAPREDUCE_-214764288_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732726118854/wal.1732726119128 (inode 16462) Holder DFSClient_NONMAPREDUCE_-214764288_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-27T16:49:15,827 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7822234{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T16:49:15,829 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8f81303{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T16:49:15,830 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T16:49:15,830 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a6d5e13{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T16:49:15,830 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f079a76{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/hadoop.log.dir/,STOPPED} 2024-11-27T16:49:15,832 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T16:49:15,833 WARN [BP-89784467-172.17.0.2-1732726112257 heartbeating to localhost/127.0.0.1:40409 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T16:49:15,833 WARN [BP-89784467-172.17.0.2-1732726112257 heartbeating to localhost/127.0.0.1:40409 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-89784467-172.17.0.2-1732726112257 (Datanode Uuid 16368b57-f612-4fd5-a59b-d4eec26c19ed) service to localhost/127.0.0.1:40409 2024-11-27T16:49:15,833 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T16:49:15,834 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/data/data5/current/BP-89784467-172.17.0.2-1732726112257 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T16:49:15,834 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/data/data6/current/BP-89784467-172.17.0.2-1732726112257 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T16:49:15,835 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T16:49:15,837 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5a400978{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T16:49:15,837 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13a44756{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T16:49:15,838 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T16:49:15,838 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46b092e1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T16:49:15,838 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ebbf344{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/hadoop.log.dir/,STOPPED} 2024-11-27T16:49:15,839 WARN [BP-89784467-172.17.0.2-1732726112257 heartbeating to localhost/127.0.0.1:40409 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T16:49:15,839 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T16:49:15,839 WARN [BP-89784467-172.17.0.2-1732726112257 heartbeating to localhost/127.0.0.1:40409 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-89784467-172.17.0.2-1732726112257 (Datanode Uuid c746d6c3-7b4e-4aa7-83e7-ac27b2137949) service to localhost/127.0.0.1:40409 2024-11-27T16:49:15,839 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T16:49:15,840 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/data/data3/current/BP-89784467-172.17.0.2-1732726112257 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T16:49:15,840 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/data/data4/current/BP-89784467-172.17.0.2-1732726112257 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T16:49:15,840 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T16:49:15,842 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@38011c53{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T16:49:15,842 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@63f56910{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T16:49:15,842 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T16:49:15,843 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cf5a85e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T16:49:15,843 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@433df981{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/hadoop.log.dir/,STOPPED} 2024-11-27T16:49:15,844 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T16:49:15,844 WARN [BP-89784467-172.17.0.2-1732726112257 heartbeating to localhost/127.0.0.1:40409 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T16:49:15,844 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T16:49:15,844 WARN [BP-89784467-172.17.0.2-1732726112257 heartbeating to localhost/127.0.0.1:40409 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-89784467-172.17.0.2-1732726112257 (Datanode Uuid bcc98693-a8fe-41ef-8eb8-7e87ebc1854e) service to localhost/127.0.0.1:40409 2024-11-27T16:49:15,845 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/data/data1/current/BP-89784467-172.17.0.2-1732726112257 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T16:49:15,845 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/cluster_9fadfa8c-d864-ab39-78a8-26f16a5e3197/data/data2/current/BP-89784467-172.17.0.2-1732726112257 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T16:49:15,845 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T16:49:15,851 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3717288f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T16:49:15,852 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T16:49:15,852 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T16:49:15,852 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T16:49:15,852 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c945047-768f-fc97-0d78-851037279652/hadoop.log.dir/,STOPPED} 2024-11-27T16:49:15,862 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T16:49:15,912 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down