2024-11-12 10:29:55,827 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-12 10:29:55,839 main DEBUG Took 0.009980 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 10:29:55,839 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 10:29:55,840 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 10:29:55,841 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 10:29:55,842 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:29:55,849 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 10:29:55,861 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:29:55,862 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:29:55,863 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:29:55,863 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:29:55,864 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:29:55,864 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:29:55,865 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:29:55,865 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:29:55,866 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:29:55,866 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:29:55,867 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:29:55,867 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:29:55,868 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:29:55,868 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:29:55,868 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:29:55,869 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:29:55,869 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:29:55,870 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:29:55,870 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:29:55,870 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:29:55,871 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:29:55,871 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:29:55,871 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:29:55,872 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 10:29:55,872 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:29:55,873 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 10:29:55,874 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 10:29:55,875 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 10:29:55,877 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 10:29:55,877 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 10:29:55,879 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 10:29:55,879 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 10:29:55,887 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 10:29:55,889 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 10:29:55,891 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 10:29:55,892 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 10:29:55,892 main DEBUG createAppenders(={Console}) 2024-11-12 10:29:55,893 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-11-12 10:29:55,893 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-12 10:29:55,894 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-11-12 10:29:55,895 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 10:29:55,895 main DEBUG OutputStream closed 2024-11-12 10:29:55,895 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 10:29:55,896 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 10:29:55,896 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-11-12 10:29:55,985 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 10:29:55,987 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 10:29:55,989 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 10:29:55,990 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 10:29:55,991 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 10:29:55,992 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 10:29:55,992 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 10:29:55,993 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 10:29:55,993 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 10:29:55,994 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 10:29:55,994 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 10:29:55,994 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 10:29:55,995 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 10:29:55,995 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 10:29:55,996 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 10:29:55,996 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 10:29:55,996 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 10:29:55,997 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 10:29:55,999 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 10:29:56,000 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-11-12 10:29:56,000 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 10:29:56,001 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-11-12T10:29:56,267 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445 2024-11-12 10:29:56,270 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 10:29:56,270 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T10:29:56,280 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-12T10:29:56,315 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T10:29:56,365 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T10:29:56,365 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T10:29:56,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-12T10:29:56,395 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436, deleteOnExit=true 2024-11-12T10:29:56,395 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-12T10:29:56,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/test.cache.data in system properties and HBase conf 2024-11-12T10:29:56,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/hadoop.tmp.dir in system properties and HBase conf 2024-11-12T10:29:56,398 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/hadoop.log.dir in system properties and HBase conf 2024-11-12T10:29:56,398 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-12T10:29:56,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-12T10:29:56,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-12T10:29:56,474 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T10:29:56,562 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T10:29:56,567 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-12T10:29:56,568 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-12T10:29:56,569 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-12T10:29:56,569 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T10:29:56,570 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-12T10:29:56,571 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-12T10:29:56,571 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T10:29:56,572 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T10:29:56,572 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-12T10:29:56,573 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/nfs.dump.dir in system properties and HBase conf 2024-11-12T10:29:56,573 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/java.io.tmpdir in system properties and HBase conf 2024-11-12T10:29:56,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T10:29:56,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-12T10:29:56,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-12T10:29:57,580 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-12T10:29:57,650 INFO [Time-limited test {}] log.Log(170): Logging initialized @2491ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-12T10:29:57,719 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T10:29:57,774 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T10:29:57,797 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T10:29:57,798 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T10:29:57,799 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T10:29:57,816 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T10:29:57,820 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/hadoop.log.dir/,AVAILABLE} 2024-11-12T10:29:57,821 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T10:29:58,002 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@398d2d49{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/java.io.tmpdir/jetty-localhost-44839-hadoop-hdfs-3_4_1-tests_jar-_-any-5547516205517805316/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T10:29:58,007 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:44839} 2024-11-12T10:29:58,008 INFO [Time-limited test {}] server.Server(415): Started @2849ms 2024-11-12T10:29:58,529 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T10:29:58,536 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T10:29:58,537 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T10:29:58,537 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T10:29:58,537 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T10:29:58,538 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29f9ab09{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/hadoop.log.dir/,AVAILABLE} 2024-11-12T10:29:58,539 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4c19a0d8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T10:29:58,638 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5bb7bef3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/java.io.tmpdir/jetty-localhost-36043-hadoop-hdfs-3_4_1-tests_jar-_-any-13667287610754358870/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T10:29:58,639 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4080a860{HTTP/1.1, (http/1.1)}{localhost:36043} 2024-11-12T10:29:58,639 INFO [Time-limited test {}] server.Server(415): Started @3480ms 2024-11-12T10:29:58,688 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T10:29:58,797 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T10:29:58,802 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T10:29:58,803 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T10:29:58,804 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T10:29:58,804 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T10:29:58,809 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fc499ca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/hadoop.log.dir/,AVAILABLE} 2024-11-12T10:29:58,810 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@152e2397{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T10:29:58,914 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@494486cb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/java.io.tmpdir/jetty-localhost-39075-hadoop-hdfs-3_4_1-tests_jar-_-any-4106475987138164899/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T10:29:58,915 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7d076b20{HTTP/1.1, (http/1.1)}{localhost:39075} 2024-11-12T10:29:58,915 INFO [Time-limited test {}] server.Server(415): Started @3756ms 2024-11-12T10:29:58,918 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T10:29:58,958 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T10:29:58,963 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T10:29:58,969 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T10:29:58,969 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T10:29:58,969 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T10:29:58,970 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@37d4eea4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/hadoop.log.dir/,AVAILABLE} 2024-11-12T10:29:58,971 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f8696d2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T10:29:59,085 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@50b13206{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/java.io.tmpdir/jetty-localhost-40767-hadoop-hdfs-3_4_1-tests_jar-_-any-6999478596903646092/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T10:29:59,085 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3267e685{HTTP/1.1, (http/1.1)}{localhost:40767} 2024-11-12T10:29:59,086 INFO [Time-limited test {}] server.Server(415): Started @3926ms 2024-11-12T10:29:59,088 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T10:30:00,514 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/data/data4/current/BP-663294313-172.17.0.3-1731407397077/current, will proceed with Du for space computation calculation, 2024-11-12T10:30:00,514 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/data/data1/current/BP-663294313-172.17.0.3-1731407397077/current, will proceed with Du for space computation calculation, 2024-11-12T10:30:00,514 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/data/data2/current/BP-663294313-172.17.0.3-1731407397077/current, will proceed with Du for space computation calculation, 2024-11-12T10:30:00,514 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/data/data3/current/BP-663294313-172.17.0.3-1731407397077/current, will proceed with Du for space computation calculation, 2024-11-12T10:30:00,547 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T10:30:00,547 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T10:30:00,596 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x333432fb7d0102e3 with lease ID 0x4740fbada0a226b7: Processing first storage report for DS-d76d6edc-a670-416d-b284-2cd56e55dc61 from datanode DatanodeRegistration(127.0.0.1:35741, datanodeUuid=45d0bcac-c90a-4a24-9c79-0339c85d65f1, infoPort=33413, infoSecurePort=0, ipcPort=45873, storageInfo=lv=-57;cid=testClusterID;nsid=1000817022;c=1731407397077) 2024-11-12T10:30:00,597 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x333432fb7d0102e3 with lease ID 0x4740fbada0a226b7: from storage DS-d76d6edc-a670-416d-b284-2cd56e55dc61 node DatanodeRegistration(127.0.0.1:35741, datanodeUuid=45d0bcac-c90a-4a24-9c79-0339c85d65f1, infoPort=33413, infoSecurePort=0, ipcPort=45873, storageInfo=lv=-57;cid=testClusterID;nsid=1000817022;c=1731407397077), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-12T10:30:00,598 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3a34b44192c442a1 with lease ID 0x4740fbada0a226b6: Processing first storage report for DS-90f90408-e2b3-455c-9019-25fea3fabe0e from datanode DatanodeRegistration(127.0.0.1:43905, datanodeUuid=702c94b9-c9ff-4905-948b-bb25cc8483bb, infoPort=39823, infoSecurePort=0, ipcPort=46857, storageInfo=lv=-57;cid=testClusterID;nsid=1000817022;c=1731407397077) 2024-11-12T10:30:00,598 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3a34b44192c442a1 with lease ID 0x4740fbada0a226b6: from storage DS-90f90408-e2b3-455c-9019-25fea3fabe0e node DatanodeRegistration(127.0.0.1:43905, datanodeUuid=702c94b9-c9ff-4905-948b-bb25cc8483bb, infoPort=39823, infoSecurePort=0, ipcPort=46857, storageInfo=lv=-57;cid=testClusterID;nsid=1000817022;c=1731407397077), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T10:30:00,599 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x333432fb7d0102e3 with lease ID 0x4740fbada0a226b7: Processing first storage report for DS-bd69db8f-0775-4c46-a769-ce9749736ee8 from datanode DatanodeRegistration(127.0.0.1:35741, datanodeUuid=45d0bcac-c90a-4a24-9c79-0339c85d65f1, infoPort=33413, infoSecurePort=0, ipcPort=45873, storageInfo=lv=-57;cid=testClusterID;nsid=1000817022;c=1731407397077) 2024-11-12T10:30:00,599 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x333432fb7d0102e3 with lease ID 0x4740fbada0a226b7: from storage DS-bd69db8f-0775-4c46-a769-ce9749736ee8 node DatanodeRegistration(127.0.0.1:35741, datanodeUuid=45d0bcac-c90a-4a24-9c79-0339c85d65f1, infoPort=33413, infoSecurePort=0, ipcPort=45873, storageInfo=lv=-57;cid=testClusterID;nsid=1000817022;c=1731407397077), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T10:30:00,599 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3a34b44192c442a1 with lease ID 0x4740fbada0a226b6: Processing first storage report for DS-06049acd-6e2b-4457-ad60-1eae3bf1986a from datanode DatanodeRegistration(127.0.0.1:43905, datanodeUuid=702c94b9-c9ff-4905-948b-bb25cc8483bb, infoPort=39823, infoSecurePort=0, ipcPort=46857, storageInfo=lv=-57;cid=testClusterID;nsid=1000817022;c=1731407397077) 2024-11-12T10:30:00,599 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3a34b44192c442a1 with lease ID 0x4740fbada0a226b6: from storage DS-06049acd-6e2b-4457-ad60-1eae3bf1986a node DatanodeRegistration(127.0.0.1:43905, datanodeUuid=702c94b9-c9ff-4905-948b-bb25cc8483bb, infoPort=39823, infoSecurePort=0, ipcPort=46857, storageInfo=lv=-57;cid=testClusterID;nsid=1000817022;c=1731407397077), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T10:30:00,696 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/data/data5/current/BP-663294313-172.17.0.3-1731407397077/current, will proceed with Du for space computation calculation, 2024-11-12T10:30:00,696 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/data/data6/current/BP-663294313-172.17.0.3-1731407397077/current, will proceed with Du for space computation calculation, 2024-11-12T10:30:00,712 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T10:30:00,717 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9dbbed8773a97eef with lease ID 0x4740fbada0a226b8: Processing first storage report for DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b from datanode DatanodeRegistration(127.0.0.1:34283, datanodeUuid=be38b4fe-5f2b-4421-a282-3500d61d9da1, infoPort=33367, infoSecurePort=0, ipcPort=36117, storageInfo=lv=-57;cid=testClusterID;nsid=1000817022;c=1731407397077) 2024-11-12T10:30:00,717 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9dbbed8773a97eef with lease ID 0x4740fbada0a226b8: from storage DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b node DatanodeRegistration(127.0.0.1:34283, datanodeUuid=be38b4fe-5f2b-4421-a282-3500d61d9da1, infoPort=33367, infoSecurePort=0, ipcPort=36117, storageInfo=lv=-57;cid=testClusterID;nsid=1000817022;c=1731407397077), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T10:30:00,717 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9dbbed8773a97eef with lease ID 0x4740fbada0a226b8: Processing first storage report for DS-63c5a43a-923d-40cf-9f7c-a2695c422e38 from datanode DatanodeRegistration(127.0.0.1:34283, datanodeUuid=be38b4fe-5f2b-4421-a282-3500d61d9da1, infoPort=33367, infoSecurePort=0, ipcPort=36117, storageInfo=lv=-57;cid=testClusterID;nsid=1000817022;c=1731407397077) 2024-11-12T10:30:00,718 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9dbbed8773a97eef with lease ID 0x4740fbada0a226b8: from storage DS-63c5a43a-923d-40cf-9f7c-a2695c422e38 node DatanodeRegistration(127.0.0.1:34283, datanodeUuid=be38b4fe-5f2b-4421-a282-3500d61d9da1, infoPort=33367, infoSecurePort=0, ipcPort=36117, storageInfo=lv=-57;cid=testClusterID;nsid=1000817022;c=1731407397077), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T10:30:00,830 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445 2024-11-12T10:30:00,891 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/zookeeper_0, clientPort=55409, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T10:30:00,899 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55409 2024-11-12T10:30:00,909 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T10:30:00,912 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T10:30:01,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741825_1001 (size=7) 2024-11-12T10:30:01,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741825_1001 (size=7) 2024-11-12T10:30:01,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741825_1001 (size=7) 2024-11-12T10:30:01,523 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf with version=8 2024-11-12T10:30:01,523 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/hbase-staging 2024-11-12T10:30:01,810 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a10fb4296bab:0 server-side Connection retries=45 2024-11-12T10:30:01,818 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T10:30:01,819 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T10:30:01,823 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T10:30:01,823 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T10:30:01,823 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T10:30:01,953 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T10:30:02,009 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T10:30:02,021 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T10:30:02,026 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T10:30:02,062 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 89274 (auto-detected) 2024-11-12T10:30:02,063 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-12T10:30:02,089 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35883 2024-11-12T10:30:02,116 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35883 connecting to ZooKeeper ensemble=127.0.0.1:55409 2024-11-12T10:30:02,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:358830x0, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T10:30:02,235 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35883-0x1012e984b560000 connected 2024-11-12T10:30:02,315 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T10:30:02,320 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T10:30:02,331 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T10:30:02,336 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf, hbase.cluster.distributed=false 2024-11-12T10:30:02,362 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T10:30:02,368 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35883 2024-11-12T10:30:02,368 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35883 2024-11-12T10:30:02,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35883 2024-11-12T10:30:02,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35883 2024-11-12T10:30:02,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35883 2024-11-12T10:30:02,469 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a10fb4296bab:0 server-side Connection retries=45 2024-11-12T10:30:02,470 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T10:30:02,470 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T10:30:02,471 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T10:30:02,471 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T10:30:02,471 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T10:30:02,473 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T10:30:02,477 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T10:30:02,478 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45327 2024-11-12T10:30:02,481 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45327 connecting to ZooKeeper ensemble=127.0.0.1:55409 2024-11-12T10:30:02,482 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T10:30:02,485 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T10:30:02,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:453270x0, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T10:30:02,494 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:453270x0, quorum=127.0.0.1:55409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T10:30:02,494 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45327-0x1012e984b560001 connected 2024-11-12T10:30:02,497 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T10:30:02,505 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T10:30:02,509 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T10:30:02,514 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T10:30:02,515 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45327 2024-11-12T10:30:02,516 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45327 2024-11-12T10:30:02,520 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45327 2024-11-12T10:30:02,520 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45327 2024-11-12T10:30:02,521 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45327 2024-11-12T10:30:02,538 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a10fb4296bab:0 server-side Connection retries=45 2024-11-12T10:30:02,538 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T10:30:02,538 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T10:30:02,539 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T10:30:02,539 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T10:30:02,539 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T10:30:02,539 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T10:30:02,539 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T10:30:02,540 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37011 2024-11-12T10:30:02,542 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37011 connecting to ZooKeeper ensemble=127.0.0.1:55409 2024-11-12T10:30:02,543 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T10:30:02,545 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T10:30:02,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:370110x0, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T10:30:02,557 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:370110x0, quorum=127.0.0.1:55409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T10:30:02,557 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37011-0x1012e984b560002 connected 2024-11-12T10:30:02,557 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T10:30:02,558 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T10:30:02,559 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37011-0x1012e984b560002, quorum=127.0.0.1:55409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T10:30:02,561 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37011-0x1012e984b560002, quorum=127.0.0.1:55409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T10:30:02,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37011 2024-11-12T10:30:02,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37011 2024-11-12T10:30:02,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37011 2024-11-12T10:30:02,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37011 2024-11-12T10:30:02,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37011 2024-11-12T10:30:02,578 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a10fb4296bab:0 server-side Connection retries=45 2024-11-12T10:30:02,578 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T10:30:02,578 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T10:30:02,578 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T10:30:02,579 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T10:30:02,579 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T10:30:02,579 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T10:30:02,579 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T10:30:02,580 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40713 2024-11-12T10:30:02,582 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40713 connecting to ZooKeeper ensemble=127.0.0.1:55409 2024-11-12T10:30:02,583 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T10:30:02,585 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T10:30:02,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:407130x0, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T10:30:02,599 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:407130x0, quorum=127.0.0.1:55409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T10:30:02,599 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40713-0x1012e984b560003 connected 2024-11-12T10:30:02,600 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T10:30:02,600 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T10:30:02,601 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T10:30:02,603 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T10:30:02,605 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40713 2024-11-12T10:30:02,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40713 2024-11-12T10:30:02,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40713 2024-11-12T10:30:02,608 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40713 2024-11-12T10:30:02,608 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40713 2024-11-12T10:30:02,623 DEBUG [M:0;a10fb4296bab:35883 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a10fb4296bab:35883 2024-11-12T10:30:02,624 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a10fb4296bab,35883,1731407401663 2024-11-12T10:30:02,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T10:30:02,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37011-0x1012e984b560002, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T10:30:02,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T10:30:02,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T10:30:02,643 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a10fb4296bab,35883,1731407401663 2024-11-12T10:30:02,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T10:30:02,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37011-0x1012e984b560002, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T10:30:02,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T10:30:02,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:02,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:02,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:02,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37011-0x1012e984b560002, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:02,674 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T10:30:02,675 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a10fb4296bab,35883,1731407401663 from backup master directory 2024-11-12T10:30:02,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a10fb4296bab,35883,1731407401663 2024-11-12T10:30:02,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T10:30:02,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T10:30:02,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T10:30:02,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37011-0x1012e984b560002, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T10:30:02,684 WARN [master/a10fb4296bab:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T10:30:02,684 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a10fb4296bab,35883,1731407401663 2024-11-12T10:30:02,686 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T10:30:02,688 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T10:30:02,738 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/hbase.id] with ID: 75eb3984-a818-4a7a-bd42-c0aa0fc13589 2024-11-12T10:30:02,738 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/.tmp/hbase.id 2024-11-12T10:30:02,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741826_1002 (size=42) 2024-11-12T10:30:02,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741826_1002 (size=42) 2024-11-12T10:30:02,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741826_1002 (size=42) 2024-11-12T10:30:02,754 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/.tmp/hbase.id]:[hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/hbase.id] 2024-11-12T10:30:02,798 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T10:30:02,804 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T10:30:02,824 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-12T10:30:02,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37011-0x1012e984b560002, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:02,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:02,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:02,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:02,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741827_1003 (size=196) 2024-11-12T10:30:02,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741827_1003 (size=196) 2024-11-12T10:30:02,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741827_1003 (size=196) 2024-11-12T10:30:02,869 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T10:30:02,871 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T10:30:02,876 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:02,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741828_1004 (size=1189) 2024-11-12T10:30:02,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741828_1004 (size=1189) 2024-11-12T10:30:02,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741828_1004 (size=1189) 2024-11-12T10:30:02,923 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store 2024-11-12T10:30:02,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741829_1005 (size=34) 2024-11-12T10:30:02,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741829_1005 (size=34) 2024-11-12T10:30:02,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741829_1005 (size=34) 2024-11-12T10:30:02,949 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T10:30:02,953 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:02,954 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T10:30:02,955 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T10:30:02,955 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T10:30:02,957 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T10:30:02,957 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T10:30:02,957 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T10:30:02,958 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731407402954Disabling compacts and flushes for region at 1731407402954Disabling writes for close at 1731407402957 (+3 ms)Writing region close event to WAL at 1731407402957Closed at 1731407402957 2024-11-12T10:30:02,961 WARN [master/a10fb4296bab:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store/.initializing 2024-11-12T10:30:02,961 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/WALs/a10fb4296bab,35883,1731407401663 2024-11-12T10:30:02,969 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T10:30:02,984 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a10fb4296bab%2C35883%2C1731407401663, suffix=, logDir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/WALs/a10fb4296bab,35883,1731407401663, archiveDir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/oldWALs, maxLogs=10 2024-11-12T10:30:03,021 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/WALs/a10fb4296bab,35883,1731407401663/a10fb4296bab%2C35883%2C1731407401663.1731407402990, exclude list is [], retry=0 2024-11-12T10:30:03,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:03,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:03,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:03,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:03,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-12T10:30:03,087 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/WALs/a10fb4296bab,35883,1731407401663/a10fb4296bab%2C35883%2C1731407401663.1731407402990 2024-11-12T10:30:03,088 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:33367:33367)] 2024-11-12T10:30:03,088 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:03,089 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:03,092 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:30:03,092 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:30:03,128 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:30:03,149 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T10:30:03,151 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:03,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:30:03,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:30:03,157 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T10:30:03,157 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:03,159 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:03,159 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:30:03,161 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T10:30:03,161 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:03,162 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:03,163 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:30:03,165 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T10:30:03,165 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:03,166 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:03,167 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:30:03,171 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:30:03,172 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:30:03,177 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:30:03,177 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:30:03,181 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T10:30:03,184 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T10:30:03,189 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:30:03,191 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70922223, jitterRate=0.056823477149009705}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T10:30:03,199 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731407403104Initializing all the Stores at 1731407403106 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731407403106Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407403107 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407403107Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407403107Cleaning up temporary data from old regions at 1731407403177 (+70 ms)Region opened successfully at 1731407403198 (+21 ms) 2024-11-12T10:30:03,200 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T10:30:03,234 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d04187c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a10fb4296bab/172.17.0.3:0 2024-11-12T10:30:03,263 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T10:30:03,272 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T10:30:03,272 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T10:30:03,275 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T10:30:03,277 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T10:30:03,282 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-12T10:30:03,282 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T10:30:03,307 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T10:30:03,315 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T10:30:03,366 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T10:30:03,369 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T10:30:03,370 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T10:30:03,377 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T10:30:03,379 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T10:30:03,382 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T10:30:03,387 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T10:30:03,388 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T10:30:03,398 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T10:30:03,413 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T10:30:03,419 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T10:30:03,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T10:30:03,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37011-0x1012e984b560002, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T10:30:03,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T10:30:03,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T10:30:03,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:03,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:03,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37011-0x1012e984b560002, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:03,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:03,433 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a10fb4296bab,35883,1731407401663, sessionid=0x1012e984b560000, setting cluster-up flag (Was=false) 2024-11-12T10:30:03,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:03,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:03,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37011-0x1012e984b560002, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:03,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:03,493 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T10:30:03,495 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a10fb4296bab,35883,1731407401663 2024-11-12T10:30:03,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:03,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37011-0x1012e984b560002, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:03,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:03,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:03,545 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T10:30:03,547 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a10fb4296bab,35883,1731407401663 2024-11-12T10:30:03,552 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T10:30:03,613 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.HRegionServer(746): ClusterId : 75eb3984-a818-4a7a-bd42-c0aa0fc13589 2024-11-12T10:30:03,613 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.HRegionServer(746): ClusterId : 75eb3984-a818-4a7a-bd42-c0aa0fc13589 2024-11-12T10:30:03,616 INFO [RS:2;a10fb4296bab:40713 {}] regionserver.HRegionServer(746): ClusterId : 75eb3984-a818-4a7a-bd42-c0aa0fc13589 2024-11-12T10:30:03,616 DEBUG [RS:1;a10fb4296bab:37011 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T10:30:03,616 DEBUG [RS:2;a10fb4296bab:40713 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T10:30:03,616 DEBUG [RS:0;a10fb4296bab:45327 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T10:30:03,617 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T10:30:03,625 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T10:30:03,631 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T10:30:03,632 DEBUG [RS:0;a10fb4296bab:45327 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T10:30:03,632 DEBUG [RS:1;a10fb4296bab:37011 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T10:30:03,632 DEBUG [RS:2;a10fb4296bab:40713 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T10:30:03,632 DEBUG [RS:0;a10fb4296bab:45327 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T10:30:03,632 DEBUG [RS:1;a10fb4296bab:37011 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T10:30:03,632 DEBUG [RS:2;a10fb4296bab:40713 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T10:30:03,637 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a10fb4296bab,35883,1731407401663 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T10:30:03,641 DEBUG [RS:1;a10fb4296bab:37011 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T10:30:03,642 DEBUG [RS:0;a10fb4296bab:45327 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T10:30:03,642 DEBUG [RS:2;a10fb4296bab:40713 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T10:30:03,642 DEBUG [RS:2;a10fb4296bab:40713 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11c200fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a10fb4296bab/172.17.0.3:0 2024-11-12T10:30:03,642 DEBUG [RS:1;a10fb4296bab:37011 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@504ab525, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a10fb4296bab/172.17.0.3:0 2024-11-12T10:30:03,642 DEBUG [RS:0;a10fb4296bab:45327 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b4df55d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a10fb4296bab/172.17.0.3:0 2024-11-12T10:30:03,644 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a10fb4296bab:0, corePoolSize=5, maxPoolSize=5 2024-11-12T10:30:03,644 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a10fb4296bab:0, corePoolSize=5, maxPoolSize=5 2024-11-12T10:30:03,644 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a10fb4296bab:0, corePoolSize=5, maxPoolSize=5 2024-11-12T10:30:03,644 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a10fb4296bab:0, corePoolSize=5, maxPoolSize=5 2024-11-12T10:30:03,644 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a10fb4296bab:0, corePoolSize=10, maxPoolSize=10 2024-11-12T10:30:03,645 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,645 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a10fb4296bab:0, corePoolSize=2, maxPoolSize=2 2024-11-12T10:30:03,645 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,655 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T10:30:03,656 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T10:30:03,658 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731407433658 2024-11-12T10:30:03,659 DEBUG [RS:1;a10fb4296bab:37011 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;a10fb4296bab:37011 2024-11-12T10:30:03,660 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T10:30:03,661 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T10:30:03,662 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T10:30:03,663 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T10:30:03,663 DEBUG [RS:1;a10fb4296bab:37011 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T10:30:03,663 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:03,663 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T10:30:03,664 DEBUG [RS:0;a10fb4296bab:45327 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a10fb4296bab:45327 2024-11-12T10:30:03,664 DEBUG [RS:2;a10fb4296bab:40713 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;a10fb4296bab:40713 2024-11-12T10:30:03,664 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T10:30:03,665 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T10:30:03,665 INFO [RS:2;a10fb4296bab:40713 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T10:30:03,665 DEBUG [RS:0;a10fb4296bab:45327 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T10:30:03,665 INFO [RS:2;a10fb4296bab:40713 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T10:30:03,665 DEBUG [RS:2;a10fb4296bab:40713 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T10:30:03,666 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.HRegionServer(2659): reportForDuty to master=a10fb4296bab,35883,1731407401663 with port=37011, startcode=1731407402537 2024-11-12T10:30:03,666 INFO [RS:2;a10fb4296bab:40713 {}] regionserver.HRegionServer(2659): reportForDuty to master=a10fb4296bab,35883,1731407401663 with port=40713, startcode=1731407402577 2024-11-12T10:30:03,666 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.HRegionServer(2659): reportForDuty to master=a10fb4296bab,35883,1731407401663 with port=45327, startcode=1731407402438 2024-11-12T10:30:03,667 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T10:30:03,668 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T10:30:03,668 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T10:30:03,668 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T10:30:03,677 DEBUG [RS:1;a10fb4296bab:37011 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T10:30:03,677 DEBUG [RS:0;a10fb4296bab:45327 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T10:30:03,677 DEBUG [RS:2;a10fb4296bab:40713 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T10:30:03,678 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,689 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T10:30:03,690 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T10:30:03,690 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T10:30:03,694 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T10:30:03,725 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T10:30:03,727 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a10fb4296bab:0:becomeActiveMaster-HFileCleaner.large.0-1731407403726,5,FailOnTimeoutGroup] 2024-11-12T10:30:03,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741831_1007 (size=1321) 2024-11-12T10:30:03,728 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a10fb4296bab:0:becomeActiveMaster-HFileCleaner.small.0-1731407403727,5,FailOnTimeoutGroup] 2024-11-12T10:30:03,728 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,728 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T10:30:03,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741831_1007 (size=1321) 2024-11-12T10:30:03,730 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741831_1007 (size=1321) 2024-11-12T10:30:03,730 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,731 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T10:30:03,732 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf 2024-11-12T10:30:03,749 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57911, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T10:30:03,749 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54561, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T10:30:03,749 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39945, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T10:30:03,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741832_1008 (size=32) 2024-11-12T10:30:03,756 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:03,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741832_1008 (size=32) 2024-11-12T10:30:03,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741832_1008 (size=32) 2024-11-12T10:30:03,757 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35883 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a10fb4296bab,40713,1731407402577 2024-11-12T10:30:03,759 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T10:30:03,760 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35883 {}] master.ServerManager(517): Registering regionserver=a10fb4296bab,40713,1731407402577 2024-11-12T10:30:03,762 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T10:30:03,762 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:03,763 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:30:03,763 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T10:30:03,766 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T10:30:03,767 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:03,768 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:30:03,768 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T10:30:03,770 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T10:30:03,771 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:03,772 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:30:03,772 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T10:30:03,773 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35883 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a10fb4296bab,37011,1731407402537 2024-11-12T10:30:03,773 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35883 {}] master.ServerManager(517): Registering regionserver=a10fb4296bab,37011,1731407402537 2024-11-12T10:30:03,776 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T10:30:03,777 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:03,777 DEBUG [RS:2;a10fb4296bab:40713 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf 2024-11-12T10:30:03,777 DEBUG [RS:2;a10fb4296bab:40713 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38657 2024-11-12T10:30:03,777 DEBUG [RS:2;a10fb4296bab:40713 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T10:30:03,778 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:30:03,778 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T10:30:03,779 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35883 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a10fb4296bab,45327,1731407402438 2024-11-12T10:30:03,779 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35883 {}] master.ServerManager(517): Registering regionserver=a10fb4296bab,45327,1731407402438 2024-11-12T10:30:03,780 DEBUG [RS:1;a10fb4296bab:37011 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf 2024-11-12T10:30:03,780 DEBUG [RS:1;a10fb4296bab:37011 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38657 2024-11-12T10:30:03,780 DEBUG [RS:1;a10fb4296bab:37011 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T10:30:03,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740 2024-11-12T10:30:03,781 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740 2024-11-12T10:30:03,784 DEBUG [RS:0;a10fb4296bab:45327 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf 2024-11-12T10:30:03,784 DEBUG [RS:0;a10fb4296bab:45327 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38657 2024-11-12T10:30:03,784 DEBUG [RS:0;a10fb4296bab:45327 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T10:30:03,785 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T10:30:03,785 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T10:30:03,786 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T10:30:03,788 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T10:30:03,791 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:30:03,792 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75351299, jitterRate=0.1228218525648117}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T10:30:03,796 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731407403757Initializing all the Stores at 1731407403758 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731407403759 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731407403759Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407403759Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731407403759Cleaning up temporary data from old regions at 1731407403785 (+26 ms)Region opened successfully at 1731407403796 (+11 ms) 2024-11-12T10:30:03,796 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T10:30:03,796 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T10:30:03,796 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T10:30:03,796 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T10:30:03,796 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T10:30:03,798 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T10:30:03,798 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731407403796Disabling compacts and flushes for region at 1731407403796Disabling writes for close at 1731407403796Writing region close event to WAL at 1731407403797 (+1 ms)Closed at 1731407403798 (+1 ms) 2024-11-12T10:30:03,800 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T10:30:03,800 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T10:30:03,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T10:30:03,815 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T10:30:03,818 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T10:30:03,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T10:30:03,840 DEBUG [RS:1;a10fb4296bab:37011 {}] zookeeper.ZKUtil(111): regionserver:37011-0x1012e984b560002, quorum=127.0.0.1:55409, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a10fb4296bab,37011,1731407402537 2024-11-12T10:30:03,840 DEBUG [RS:2;a10fb4296bab:40713 {}] zookeeper.ZKUtil(111): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a10fb4296bab,40713,1731407402577 2024-11-12T10:30:03,840 WARN [RS:2;a10fb4296bab:40713 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T10:30:03,840 WARN [RS:1;a10fb4296bab:37011 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T10:30:03,840 INFO [RS:2;a10fb4296bab:40713 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:03,840 INFO [RS:1;a10fb4296bab:37011 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:03,841 DEBUG [RS:2;a10fb4296bab:40713 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,40713,1731407402577 2024-11-12T10:30:03,841 DEBUG [RS:1;a10fb4296bab:37011 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537 2024-11-12T10:30:03,841 DEBUG [RS:0;a10fb4296bab:45327 {}] zookeeper.ZKUtil(111): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a10fb4296bab,45327,1731407402438 2024-11-12T10:30:03,841 WARN [RS:0;a10fb4296bab:45327 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T10:30:03,841 INFO [RS:0;a10fb4296bab:45327 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:03,841 DEBUG [RS:0;a10fb4296bab:45327 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,45327,1731407402438 2024-11-12T10:30:03,842 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a10fb4296bab,40713,1731407402577] 2024-11-12T10:30:03,842 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a10fb4296bab,37011,1731407402537] 2024-11-12T10:30:03,842 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a10fb4296bab,45327,1731407402438] 2024-11-12T10:30:03,865 INFO [RS:2;a10fb4296bab:40713 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T10:30:03,865 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T10:30:03,865 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T10:30:03,882 INFO [RS:2;a10fb4296bab:40713 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T10:30:03,882 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T10:30:03,882 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T10:30:03,887 INFO [RS:0;a10fb4296bab:45327 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T10:30:03,887 INFO [RS:2;a10fb4296bab:40713 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T10:30:03,887 INFO [RS:1;a10fb4296bab:37011 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T10:30:03,887 INFO [RS:0;a10fb4296bab:45327 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,887 INFO [RS:2;a10fb4296bab:40713 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,887 INFO [RS:1;a10fb4296bab:37011 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,888 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T10:30:03,888 INFO [RS:2;a10fb4296bab:40713 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T10:30:03,888 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T10:30:03,894 INFO [RS:2;a10fb4296bab:40713 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T10:30:03,894 INFO [RS:0;a10fb4296bab:45327 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T10:30:03,894 INFO [RS:1;a10fb4296bab:37011 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T10:30:03,895 INFO [RS:2;a10fb4296bab:40713 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,895 INFO [RS:0;a10fb4296bab:45327 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,895 INFO [RS:1;a10fb4296bab:37011 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,896 DEBUG [RS:2;a10fb4296bab:40713 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,896 DEBUG [RS:1;a10fb4296bab:37011 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,896 DEBUG [RS:2;a10fb4296bab:40713 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,896 DEBUG [RS:1;a10fb4296bab:37011 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,896 DEBUG [RS:2;a10fb4296bab:40713 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,896 DEBUG [RS:1;a10fb4296bab:37011 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,896 DEBUG [RS:2;a10fb4296bab:40713 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,896 DEBUG [RS:1;a10fb4296bab:37011 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,896 DEBUG [RS:2;a10fb4296bab:40713 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,896 DEBUG [RS:1;a10fb4296bab:37011 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,896 DEBUG [RS:2;a10fb4296bab:40713 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0, corePoolSize=2, maxPoolSize=2 2024-11-12T10:30:03,896 DEBUG [RS:1;a10fb4296bab:37011 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0, corePoolSize=2, maxPoolSize=2 2024-11-12T10:30:03,896 DEBUG [RS:2;a10fb4296bab:40713 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,896 DEBUG [RS:1;a10fb4296bab:37011 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,896 DEBUG [RS:1;a10fb4296bab:37011 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,896 DEBUG [RS:2;a10fb4296bab:40713 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,897 DEBUG [RS:1;a10fb4296bab:37011 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,897 DEBUG [RS:2;a10fb4296bab:40713 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,897 DEBUG [RS:1;a10fb4296bab:37011 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,897 DEBUG [RS:2;a10fb4296bab:40713 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,897 DEBUG [RS:1;a10fb4296bab:37011 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,897 DEBUG [RS:2;a10fb4296bab:40713 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,897 DEBUG [RS:1;a10fb4296bab:37011 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,897 DEBUG [RS:2;a10fb4296bab:40713 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,897 DEBUG [RS:1;a10fb4296bab:37011 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a10fb4296bab:0, corePoolSize=3, maxPoolSize=3 2024-11-12T10:30:03,897 DEBUG [RS:2;a10fb4296bab:40713 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a10fb4296bab:0, corePoolSize=3, maxPoolSize=3 2024-11-12T10:30:03,897 DEBUG [RS:1;a10fb4296bab:37011 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a10fb4296bab:0, corePoolSize=3, maxPoolSize=3 2024-11-12T10:30:03,897 DEBUG [RS:2;a10fb4296bab:40713 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a10fb4296bab:0, corePoolSize=3, maxPoolSize=3 2024-11-12T10:30:03,896 DEBUG [RS:0;a10fb4296bab:45327 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,897 DEBUG [RS:0;a10fb4296bab:45327 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,898 DEBUG [RS:0;a10fb4296bab:45327 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,898 DEBUG [RS:0;a10fb4296bab:45327 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,898 DEBUG [RS:0;a10fb4296bab:45327 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,898 DEBUG [RS:0;a10fb4296bab:45327 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0, corePoolSize=2, maxPoolSize=2 2024-11-12T10:30:03,898 DEBUG [RS:0;a10fb4296bab:45327 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,898 DEBUG [RS:0;a10fb4296bab:45327 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,898 DEBUG [RS:0;a10fb4296bab:45327 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,898 DEBUG [RS:0;a10fb4296bab:45327 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,898 DEBUG [RS:0;a10fb4296bab:45327 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,898 DEBUG [RS:0;a10fb4296bab:45327 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a10fb4296bab:0, corePoolSize=1, maxPoolSize=1 2024-11-12T10:30:03,899 DEBUG [RS:0;a10fb4296bab:45327 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a10fb4296bab:0, corePoolSize=3, maxPoolSize=3 2024-11-12T10:30:03,899 DEBUG [RS:0;a10fb4296bab:45327 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a10fb4296bab:0, corePoolSize=3, maxPoolSize=3 2024-11-12T10:30:03,901 INFO [RS:2;a10fb4296bab:40713 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,901 INFO [RS:1;a10fb4296bab:37011 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,901 INFO [RS:2;a10fb4296bab:40713 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,901 INFO [RS:0;a10fb4296bab:45327 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,901 INFO [RS:1;a10fb4296bab:37011 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,902 INFO [RS:2;a10fb4296bab:40713 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,902 INFO [RS:0;a10fb4296bab:45327 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,902 INFO [RS:1;a10fb4296bab:37011 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,902 INFO [RS:2;a10fb4296bab:40713 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,902 INFO [RS:0;a10fb4296bab:45327 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,902 INFO [RS:1;a10fb4296bab:37011 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,902 INFO [RS:2;a10fb4296bab:40713 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,902 INFO [RS:0;a10fb4296bab:45327 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,902 INFO [RS:1;a10fb4296bab:37011 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,902 INFO [RS:2;a10fb4296bab:40713 {}] hbase.ChoreService(168): Chore ScheduledChore name=a10fb4296bab,40713,1731407402577-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T10:30:03,902 INFO [RS:1;a10fb4296bab:37011 {}] hbase.ChoreService(168): Chore ScheduledChore name=a10fb4296bab,37011,1731407402537-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T10:30:03,902 INFO [RS:0;a10fb4296bab:45327 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,902 INFO [RS:0;a10fb4296bab:45327 {}] hbase.ChoreService(168): Chore ScheduledChore name=a10fb4296bab,45327,1731407402438-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T10:30:03,926 INFO [RS:2;a10fb4296bab:40713 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T10:30:03,926 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T10:30:03,926 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T10:30:03,929 INFO [RS:2;a10fb4296bab:40713 {}] hbase.ChoreService(168): Chore ScheduledChore name=a10fb4296bab,40713,1731407402577-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,929 INFO [RS:1;a10fb4296bab:37011 {}] hbase.ChoreService(168): Chore ScheduledChore name=a10fb4296bab,37011,1731407402537-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,929 INFO [RS:0;a10fb4296bab:45327 {}] hbase.ChoreService(168): Chore ScheduledChore name=a10fb4296bab,45327,1731407402438-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,929 INFO [RS:2;a10fb4296bab:40713 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,929 INFO [RS:1;a10fb4296bab:37011 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,929 INFO [RS:0;a10fb4296bab:45327 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,929 INFO [RS:2;a10fb4296bab:40713 {}] regionserver.Replication(171): a10fb4296bab,40713,1731407402577 started 2024-11-12T10:30:03,929 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.Replication(171): a10fb4296bab,37011,1731407402537 started 2024-11-12T10:30:03,929 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.Replication(171): a10fb4296bab,45327,1731407402438 started 2024-11-12T10:30:03,955 INFO [RS:1;a10fb4296bab:37011 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,955 INFO [RS:0;a10fb4296bab:45327 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,955 INFO [RS:2;a10fb4296bab:40713 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:03,955 INFO [RS:2;a10fb4296bab:40713 {}] regionserver.HRegionServer(1482): Serving as a10fb4296bab,40713,1731407402577, RpcServer on a10fb4296bab/172.17.0.3:40713, sessionid=0x1012e984b560003 2024-11-12T10:30:03,955 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.HRegionServer(1482): Serving as a10fb4296bab,37011,1731407402537, RpcServer on a10fb4296bab/172.17.0.3:37011, sessionid=0x1012e984b560002 2024-11-12T10:30:03,955 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.HRegionServer(1482): Serving as a10fb4296bab,45327,1731407402438, RpcServer on a10fb4296bab/172.17.0.3:45327, sessionid=0x1012e984b560001 2024-11-12T10:30:03,956 DEBUG [RS:0;a10fb4296bab:45327 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T10:30:03,956 DEBUG [RS:2;a10fb4296bab:40713 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T10:30:03,956 DEBUG [RS:1;a10fb4296bab:37011 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T10:30:03,956 DEBUG [RS:1;a10fb4296bab:37011 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a10fb4296bab,37011,1731407402537 2024-11-12T10:30:03,956 DEBUG [RS:2;a10fb4296bab:40713 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a10fb4296bab,40713,1731407402577 2024-11-12T10:30:03,956 DEBUG [RS:0;a10fb4296bab:45327 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a10fb4296bab,45327,1731407402438 2024-11-12T10:30:03,957 DEBUG [RS:1;a10fb4296bab:37011 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a10fb4296bab,37011,1731407402537' 2024-11-12T10:30:03,957 DEBUG [RS:2;a10fb4296bab:40713 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a10fb4296bab,40713,1731407402577' 2024-11-12T10:30:03,957 DEBUG [RS:0;a10fb4296bab:45327 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a10fb4296bab,45327,1731407402438' 2024-11-12T10:30:03,957 DEBUG [RS:1;a10fb4296bab:37011 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T10:30:03,957 DEBUG [RS:0;a10fb4296bab:45327 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T10:30:03,957 DEBUG [RS:2;a10fb4296bab:40713 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T10:30:03,958 DEBUG [RS:0;a10fb4296bab:45327 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T10:30:03,958 DEBUG [RS:2;a10fb4296bab:40713 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T10:30:03,958 DEBUG [RS:1;a10fb4296bab:37011 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T10:30:03,958 DEBUG [RS:0;a10fb4296bab:45327 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T10:30:03,958 DEBUG [RS:2;a10fb4296bab:40713 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T10:30:03,959 DEBUG [RS:0;a10fb4296bab:45327 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T10:30:03,959 DEBUG [RS:2;a10fb4296bab:40713 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T10:30:03,959 DEBUG [RS:1;a10fb4296bab:37011 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T10:30:03,959 DEBUG [RS:1;a10fb4296bab:37011 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T10:30:03,959 DEBUG [RS:0;a10fb4296bab:45327 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a10fb4296bab,45327,1731407402438 2024-11-12T10:30:03,959 DEBUG [RS:2;a10fb4296bab:40713 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a10fb4296bab,40713,1731407402577 2024-11-12T10:30:03,959 DEBUG [RS:0;a10fb4296bab:45327 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a10fb4296bab,45327,1731407402438' 2024-11-12T10:30:03,959 DEBUG [RS:1;a10fb4296bab:37011 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a10fb4296bab,37011,1731407402537 2024-11-12T10:30:03,959 DEBUG [RS:2;a10fb4296bab:40713 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a10fb4296bab,40713,1731407402577' 2024-11-12T10:30:03,959 DEBUG [RS:0;a10fb4296bab:45327 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T10:30:03,959 DEBUG [RS:1;a10fb4296bab:37011 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a10fb4296bab,37011,1731407402537' 2024-11-12T10:30:03,959 DEBUG [RS:2;a10fb4296bab:40713 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T10:30:03,959 DEBUG [RS:1;a10fb4296bab:37011 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T10:30:03,960 DEBUG [RS:0;a10fb4296bab:45327 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T10:30:03,960 DEBUG [RS:2;a10fb4296bab:40713 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T10:30:03,960 DEBUG [RS:0;a10fb4296bab:45327 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T10:30:03,960 INFO [RS:0;a10fb4296bab:45327 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T10:30:03,960 DEBUG [RS:2;a10fb4296bab:40713 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T10:30:03,960 INFO [RS:0;a10fb4296bab:45327 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T10:30:03,960 INFO [RS:2;a10fb4296bab:40713 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T10:30:03,960 INFO [RS:2;a10fb4296bab:40713 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T10:30:03,965 DEBUG [RS:1;a10fb4296bab:37011 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T10:30:03,966 DEBUG [RS:1;a10fb4296bab:37011 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T10:30:03,966 INFO [RS:1;a10fb4296bab:37011 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T10:30:03,966 INFO [RS:1;a10fb4296bab:37011 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T10:30:03,969 WARN [a10fb4296bab:35883 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-12T10:30:04,066 INFO [RS:0;a10fb4296bab:45327 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T10:30:04,066 INFO [RS:2;a10fb4296bab:40713 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T10:30:04,067 INFO [RS:1;a10fb4296bab:37011 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T10:30:04,069 INFO [RS:0;a10fb4296bab:45327 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a10fb4296bab%2C45327%2C1731407402438, suffix=, logDir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,45327,1731407402438, archiveDir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/oldWALs, maxLogs=32 2024-11-12T10:30:04,069 INFO [RS:2;a10fb4296bab:40713 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a10fb4296bab%2C40713%2C1731407402577, suffix=, logDir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,40713,1731407402577, archiveDir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/oldWALs, maxLogs=32 2024-11-12T10:30:04,069 INFO [RS:1;a10fb4296bab:37011 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a10fb4296bab%2C37011%2C1731407402537, suffix=, logDir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537, archiveDir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/oldWALs, maxLogs=32 2024-11-12T10:30:04,085 DEBUG [RS:2;a10fb4296bab:40713 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,40713,1731407402577/a10fb4296bab%2C40713%2C1731407402577.1731407404072, exclude list is [], retry=0 2024-11-12T10:30:04,086 DEBUG [RS:1;a10fb4296bab:37011 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537/a10fb4296bab%2C37011%2C1731407402537.1731407404072, exclude list is [], retry=0 2024-11-12T10:30:04,087 DEBUG [RS:0;a10fb4296bab:45327 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,45327,1731407402438/a10fb4296bab%2C45327%2C1731407402438.1731407404072, exclude list is [], retry=0 2024-11-12T10:30:04,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:04,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:04,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:04,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:04,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:04,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:04,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:04,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:04,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:04,111 INFO [RS:0;a10fb4296bab:45327 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,45327,1731407402438/a10fb4296bab%2C45327%2C1731407402438.1731407404072 2024-11-12T10:30:04,112 INFO [RS:1;a10fb4296bab:37011 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537/a10fb4296bab%2C37011%2C1731407402537.1731407404072 2024-11-12T10:30:04,116 DEBUG [RS:0;a10fb4296bab:45327 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33413:33413)] 2024-11-12T10:30:04,116 DEBUG [RS:1;a10fb4296bab:37011 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:33367:33367)] 2024-11-12T10:30:04,118 INFO [RS:2;a10fb4296bab:40713 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,40713,1731407402577/a10fb4296bab%2C40713%2C1731407402577.1731407404072 2024-11-12T10:30:04,120 DEBUG [RS:2;a10fb4296bab:40713 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33367:33367)] 2024-11-12T10:30:04,221 DEBUG [a10fb4296bab:35883 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-12T10:30:04,228 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(204): Hosts are {a10fb4296bab=0} racks are {/default-rack=0} 2024-11-12T10:30:04,234 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T10:30:04,235 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T10:30:04,235 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-12T10:30:04,235 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T10:30:04,235 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T10:30:04,235 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-12T10:30:04,235 INFO [a10fb4296bab:35883 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T10:30:04,235 INFO [a10fb4296bab:35883 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T10:30:04,235 INFO [a10fb4296bab:35883 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-12T10:30:04,235 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T10:30:04,243 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a10fb4296bab,37011,1731407402537 2024-11-12T10:30:04,249 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a10fb4296bab,37011,1731407402537, state=OPENING 2024-11-12T10:30:04,303 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T10:30:04,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37011-0x1012e984b560002, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:04,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:04,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:04,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:04,315 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T10:30:04,315 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T10:30:04,315 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T10:30:04,315 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T10:30:04,316 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T10:30:04,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a10fb4296bab,37011,1731407402537}] 2024-11-12T10:30:04,488 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T10:30:04,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34015, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T10:30:04,502 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T10:30:04,503 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:04,503 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T10:30:04,506 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a10fb4296bab%2C37011%2C1731407402537.meta, suffix=.meta, logDir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537, archiveDir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/oldWALs, maxLogs=32 2024-11-12T10:30:04,520 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537/a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta, exclude list is [], retry=0 2024-11-12T10:30:04,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:04,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:04,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:04,528 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537/a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta 2024-11-12T10:30:04,528 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33413:33413)] 2024-11-12T10:30:04,528 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:04,530 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T10:30:04,532 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T10:30:04,536 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T10:30:04,540 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T10:30:04,541 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:04,541 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T10:30:04,541 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T10:30:04,544 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T10:30:04,546 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T10:30:04,546 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:04,547 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:30:04,547 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T10:30:04,548 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T10:30:04,548 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:04,549 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:30:04,549 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T10:30:04,551 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T10:30:04,551 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:04,552 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:30:04,552 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T10:30:04,553 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T10:30:04,553 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:04,554 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:30:04,554 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T10:30:04,556 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740 2024-11-12T10:30:04,559 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740 2024-11-12T10:30:04,562 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T10:30:04,562 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T10:30:04,563 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T10:30:04,566 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T10:30:04,568 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66576091, jitterRate=-0.007938936352729797}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T10:30:04,568 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T10:30:04,570 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731407404542Writing region info on filesystem at 1731407404542Initializing all the Stores at 1731407404544 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731407404544Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731407404544Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407404544Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731407404544Cleaning up temporary data from old regions at 1731407404562 (+18 ms)Running coprocessor post-open hooks at 1731407404568 (+6 ms)Region opened successfully at 1731407404570 (+2 ms) 2024-11-12T10:30:04,576 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731407404481 2024-11-12T10:30:04,588 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T10:30:04,589 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T10:30:04,590 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a10fb4296bab,37011,1731407402537 2024-11-12T10:30:04,592 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a10fb4296bab,37011,1731407402537, state=OPEN 2024-11-12T10:30:04,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37011-0x1012e984b560002, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T10:30:04,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T10:30:04,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T10:30:04,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T10:30:04,598 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T10:30:04,598 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T10:30:04,598 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T10:30:04,598 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T10:30:04,599 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a10fb4296bab,37011,1731407402537 2024-11-12T10:30:04,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T10:30:04,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a10fb4296bab,37011,1731407402537 in 281 msec 2024-11-12T10:30:04,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T10:30:04,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 800 msec 2024-11-12T10:30:04,612 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T10:30:04,612 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T10:30:04,628 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:30:04,629 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1] 2024-11-12T10:30:04,648 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:30:04,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42911, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:30:04,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0910 sec 2024-11-12T10:30:04,672 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731407404672, completionTime=-1 2024-11-12T10:30:04,676 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-12T10:30:04,676 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T10:30:04,703 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-12T10:30:04,703 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731407464703 2024-11-12T10:30:04,703 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731407524703 2024-11-12T10:30:04,703 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 27 msec 2024-11-12T10:30:04,705 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-12T10:30:04,713 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a10fb4296bab,35883,1731407401663-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:04,713 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a10fb4296bab,35883,1731407401663-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:04,713 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a10fb4296bab,35883,1731407401663-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:04,715 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a10fb4296bab:35883, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:04,715 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:04,716 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:04,722 DEBUG [master/a10fb4296bab:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T10:30:04,746 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.062sec 2024-11-12T10:30:04,749 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T10:30:04,750 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T10:30:04,751 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T10:30:04,751 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T10:30:04,752 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T10:30:04,752 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a10fb4296bab,35883,1731407401663-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T10:30:04,753 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a10fb4296bab,35883,1731407401663-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T10:30:04,757 DEBUG [master/a10fb4296bab:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T10:30:04,759 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T10:30:04,759 INFO [master/a10fb4296bab:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a10fb4296bab,35883,1731407401663-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:04,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@181f9e34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:30:04,824 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a10fb4296bab,35883,-1 for getting cluster id 2024-11-12T10:30:04,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T10:30:04,835 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75eb3984-a818-4a7a-bd42-c0aa0fc13589' 2024-11-12T10:30:04,838 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T10:30:04,838 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75eb3984-a818-4a7a-bd42-c0aa0fc13589" 2024-11-12T10:30:04,838 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4289cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:30:04,839 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a10fb4296bab,35883,-1] 2024-11-12T10:30:04,841 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T10:30:04,844 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:30:04,845 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T10:30:04,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76a2fdee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T10:30:04,848 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:30:04,857 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1] 2024-11-12T10:30:04,858 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:30:04,861 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:30:04,887 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a10fb4296bab,35883,1731407401663 2024-11-12T10:30:04,887 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:38657/hbase 2024-11-12T10:30:04,904 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=364, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=849, ProcessCount=11, AvailableMemoryMB=7750 2024-11-12T10:30:04,923 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:04,927 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:04,928 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T10:30:04,933 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-82439526, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/hregion-82439526, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:04,947 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-82439526/hregion-82439526.1731407404935, exclude list is [], retry=0 2024-11-12T10:30:04,952 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:04,952 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:04,952 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:04,957 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-82439526/hregion-82439526.1731407404935 2024-11-12T10:30:04,958 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:39823:39823)] 2024-11-12T10:30:04,958 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6c82b3e06d351a80e22e7e674f061772, NAME => 'testReplayEditsWrittenIntoWAL,,1731407404924.6c82b3e06d351a80e22e7e674f061772.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38657/hbase 2024-11-12T10:30:04,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741838_1014 (size=64) 2024-11-12T10:30:04,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741838_1014 (size=64) 2024-11-12T10:30:04,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741838_1014 (size=64) 2024-11-12T10:30:04,977 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731407404924.6c82b3e06d351a80e22e7e674f061772.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:04,980 INFO [StoreOpener-6c82b3e06d351a80e22e7e674f061772-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6c82b3e06d351a80e22e7e674f061772 2024-11-12T10:30:04,982 INFO [StoreOpener-6c82b3e06d351a80e22e7e674f061772-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c82b3e06d351a80e22e7e674f061772 columnFamilyName a 2024-11-12T10:30:04,982 DEBUG [StoreOpener-6c82b3e06d351a80e22e7e674f061772-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:04,983 INFO [StoreOpener-6c82b3e06d351a80e22e7e674f061772-1 {}] regionserver.HStore(327): Store=6c82b3e06d351a80e22e7e674f061772/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:04,983 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6c82b3e06d351a80e22e7e674f061772 2024-11-12T10:30:04,985 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772 2024-11-12T10:30:04,985 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772 2024-11-12T10:30:04,986 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6c82b3e06d351a80e22e7e674f061772 2024-11-12T10:30:04,986 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6c82b3e06d351a80e22e7e674f061772 2024-11-12T10:30:04,989 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6c82b3e06d351a80e22e7e674f061772 2024-11-12T10:30:04,993 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:30:04,994 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6c82b3e06d351a80e22e7e674f061772; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67952166, jitterRate=0.012566179037094116}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T10:30:04,995 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6c82b3e06d351a80e22e7e674f061772: Writing region info on filesystem at 1731407404977Initializing all the Stores at 1731407404979 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407404979Cleaning up temporary data from old regions at 1731407404986 (+7 ms)Region opened successfully at 1731407404995 (+9 ms) 2024-11-12T10:30:04,995 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6c82b3e06d351a80e22e7e674f061772, disabling compactions & flushes 2024-11-12T10:30:04,995 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731407404924.6c82b3e06d351a80e22e7e674f061772. 2024-11-12T10:30:04,995 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731407404924.6c82b3e06d351a80e22e7e674f061772. 2024-11-12T10:30:04,995 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731407404924.6c82b3e06d351a80e22e7e674f061772. after waiting 0 ms 2024-11-12T10:30:04,995 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731407404924.6c82b3e06d351a80e22e7e674f061772. 2024-11-12T10:30:04,996 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731407404924.6c82b3e06d351a80e22e7e674f061772. 2024-11-12T10:30:04,996 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6c82b3e06d351a80e22e7e674f061772: Waiting for close lock at 1731407404995Disabling compacts and flushes for region at 1731407404995Disabling writes for close at 1731407404995Writing region close event to WAL at 1731407404996 (+1 ms)Closed at 1731407404996 2024-11-12T10:30:05,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741837_1013 (size=93) 2024-11-12T10:30:05,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741837_1013 (size=93) 2024-11-12T10:30:05,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741837_1013 (size=93) 2024-11-12T10:30:05,013 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T10:30:05,014 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-82439526:(num 1731407404935) 2024-11-12T10:30:05,016 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-12T10:30:05,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741839_1015 (size=369) 2024-11-12T10:30:05,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741839_1015 (size=369) 2024-11-12T10:30:05,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741839_1015 (size=369) 2024-11-12T10:30:05,043 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-12T10:30:05,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741840_1016 (size=231) 2024-11-12T10:30:05,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741840_1016 (size=231) 2024-11-12T10:30:05,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741840_1016 (size=231) 2024-11-12T10:30:05,083 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-1, size=369 (369bytes) 2024-11-12T10:30:05,084 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-12T10:30:05,084 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-12T10:30:05,085 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-1 2024-11-12T10:30:05,091 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-1 after 3ms 2024-11-12T10:30:05,099 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-1 took 17ms 2024-11-12T10:30:05,104 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-1 so closing down 2024-11-12T10:30:05,108 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-12T10:30:05,109 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T10:30:05,110 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/recovered.edits/0000000000000000001-wal-1.temp 2024-11-12T10:30:05,110 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T10:30:05,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741841_1017 (size=369) 2024-11-12T10:30:05,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741841_1017 (size=369) 2024-11-12T10:30:05,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741841_1017 (size=369) 2024-11-12T10:30:05,126 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-12T10:30:05,129 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/recovered.edits/0000000000000000002 2024-11-12T10:30:05,134 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 29 ms; skipped=0; WAL=hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-12T10:30:05,134 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-1, journal: Splitting hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-1, size=369 (369bytes) at 1731407405083Finishing writing output for hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-1 so closing down at 1731407405104 (+21 ms)Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/recovered.edits/0000000000000000001-wal-1.temp at 1731407405110 (+6 ms)3 split writer threads finished at 1731407405111 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731407405126 (+15 ms)Rename recovered edits hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/recovered.edits/0000000000000000002 at 1731407405129 (+3 ms)Processed 2 edits across 1 Regions in 29 ms; skipped=0; WAL=hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1731407405134 (+5 ms) 2024-11-12T10:30:05,157 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-2, size=231 (231bytes) 2024-11-12T10:30:05,157 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-2 2024-11-12T10:30:05,158 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-2 after 1ms 2024-11-12T10:30:05,163 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-2 took 7ms 2024-11-12T10:30:05,166 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-2 so closing down 2024-11-12T10:30:05,166 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T10:30:05,169 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-12T10:30:05,170 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/recovered.edits/0000000000000000002-wal-2.temp 2024-11-12T10:30:05,171 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T10:30:05,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741842_1018 (size=231) 2024-11-12T10:30:05,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741842_1018 (size=231) 2024-11-12T10:30:05,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741842_1018 (size=231) 2024-11-12T10:30:05,184 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-12T10:30:05,192 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-11-12T10:30:05,193 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 30 ms; skipped=0; WAL=hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-12T10:30:05,194 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-2, journal: Splitting hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-2, size=231 (231bytes) at 1731407405157Finishing writing output for hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-2 so closing down at 1731407405166 (+9 ms)Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/recovered.edits/0000000000000000002-wal-2.temp at 1731407405170 (+4 ms)3 split writer threads finished at 1731407405171 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731407405184 (+13 ms)Processed 1 edits across 1 Regions in 30 ms; skipped=0; WAL=hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1731407405193 (+9 ms) 2024-11-12T10:30:05,194 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:05,196 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:05,211 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal.1731407405197, exclude list is [], retry=0 2024-11-12T10:30:05,215 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:05,216 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:05,216 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:05,219 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal.1731407405197 2024-11-12T10:30:05,219 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:33413:33413)] 2024-11-12T10:30:05,219 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c82b3e06d351a80e22e7e674f061772, NAME => 'testReplayEditsWrittenIntoWAL,,1731407404924.6c82b3e06d351a80e22e7e674f061772.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:05,220 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731407404924.6c82b3e06d351a80e22e7e674f061772.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:05,220 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6c82b3e06d351a80e22e7e674f061772 2024-11-12T10:30:05,220 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6c82b3e06d351a80e22e7e674f061772 2024-11-12T10:30:05,222 INFO [StoreOpener-6c82b3e06d351a80e22e7e674f061772-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6c82b3e06d351a80e22e7e674f061772 2024-11-12T10:30:05,224 INFO [StoreOpener-6c82b3e06d351a80e22e7e674f061772-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c82b3e06d351a80e22e7e674f061772 columnFamilyName a 2024-11-12T10:30:05,225 DEBUG [StoreOpener-6c82b3e06d351a80e22e7e674f061772-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:05,226 INFO [StoreOpener-6c82b3e06d351a80e22e7e674f061772-1 {}] regionserver.HStore(327): Store=6c82b3e06d351a80e22e7e674f061772/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:05,226 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6c82b3e06d351a80e22e7e674f061772 2024-11-12T10:30:05,228 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772 2024-11-12T10:30:05,231 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772 2024-11-12T10:30:05,232 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/recovered.edits/0000000000000000002 2024-11-12T10:30:05,242 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/recovered.edits/0000000000000000002 2024-11-12T10:30:05,247 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6c82b3e06d351a80e22e7e674f061772 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-12T10:30:05,305 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/.tmp/a/1ff7c6803aa64302bc75a2a816c81848 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731407405014/Put/seqid=0 2024-11-12T10:30:05,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741844_1020 (size=5170) 2024-11-12T10:30:05,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741844_1020 (size=5170) 2024-11-12T10:30:05,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741844_1020 (size=5170) 2024-11-12T10:30:05,317 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/.tmp/a/1ff7c6803aa64302bc75a2a816c81848 2024-11-12T10:30:05,356 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/.tmp/a/1ff7c6803aa64302bc75a2a816c81848 as hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/a/1ff7c6803aa64302bc75a2a816c81848 2024-11-12T10:30:05,368 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/a/1ff7c6803aa64302bc75a2a816c81848, entries=2, sequenceid=2, filesize=5.0 K 2024-11-12T10:30:05,375 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 6c82b3e06d351a80e22e7e674f061772 in 128ms, sequenceid=2, compaction requested=false; wal=null 2024-11-12T10:30:05,377 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/recovered.edits/0000000000000000002 2024-11-12T10:30:05,378 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6c82b3e06d351a80e22e7e674f061772 2024-11-12T10:30:05,378 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6c82b3e06d351a80e22e7e674f061772 2024-11-12T10:30:05,382 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6c82b3e06d351a80e22e7e674f061772 2024-11-12T10:30:05,386 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/6c82b3e06d351a80e22e7e674f061772/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-12T10:30:05,388 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6c82b3e06d351a80e22e7e674f061772; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66682218, jitterRate=-0.006357520818710327}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T10:30:05,390 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6c82b3e06d351a80e22e7e674f061772: Writing region info on filesystem at 1731407405220Initializing all the Stores at 1731407405222 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407405222Obtaining lock to block concurrent updates at 1731407405247 (+25 ms)Preparing flush snapshotting stores in 6c82b3e06d351a80e22e7e674f061772 at 1731407405247Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731407404924.6c82b3e06d351a80e22e7e674f061772., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731407405251 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731407404924.6c82b3e06d351a80e22e7e674f061772. at 1731407405251Flushing 6c82b3e06d351a80e22e7e674f061772/a: creating writer at 1731407405253 (+2 ms)Flushing 6c82b3e06d351a80e22e7e674f061772/a: appending metadata at 1731407405293 (+40 ms)Flushing 6c82b3e06d351a80e22e7e674f061772/a: closing flushed file at 1731407405296 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49442b8: reopening flushed file at 1731407405355 (+59 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 6c82b3e06d351a80e22e7e674f061772 in 128ms, sequenceid=2, compaction requested=false; wal=null at 1731407405375 (+20 ms)Cleaning up temporary data from old regions at 1731407405378 (+3 ms)Region opened successfully at 1731407405390 (+12 ms) 2024-11-12T10:30:05,453 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=376 (was 364) Potentially hanging thread: PacketResponder: BP-663294313-172.17.0.3-1731407397077:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:48838 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-663294313-172.17.0.3-1731407397077:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:35664 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-663294313-172.17.0.3-1731407397077:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:48966 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:49184 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:49304 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:38657/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:35638 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=849 (was 849), ProcessCount=11 (was 11), AvailableMemoryMB=7646 (was 7750) 2024-11-12T10:30:05,468 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=376, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=849, ProcessCount=11, AvailableMemoryMB=7645 2024-11-12T10:30:05,490 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:05,493 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:05,494 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T10:30:05,498 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-45596041, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/hregion-45596041, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:05,511 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-45596041/hregion-45596041.1731407405499, exclude list is [], retry=0 2024-11-12T10:30:05,515 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:05,515 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:05,516 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:05,521 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-45596041/hregion-45596041.1731407405499 2024-11-12T10:30:05,522 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:39823:39823)] 2024-11-12T10:30:05,522 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7f47811997514991aba525c2f71bf84c, NAME => 'testReplayEditsWrittenIntoWAL,,1731407405491.7f47811997514991aba525c2f71bf84c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38657/hbase 2024-11-12T10:30:05,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741846_1022 (size=64) 2024-11-12T10:30:05,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741846_1022 (size=64) 2024-11-12T10:30:05,544 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731407405491.7f47811997514991aba525c2f71bf84c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:05,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741846_1022 (size=64) 2024-11-12T10:30:05,547 INFO [StoreOpener-7f47811997514991aba525c2f71bf84c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7f47811997514991aba525c2f71bf84c 2024-11-12T10:30:05,549 INFO [StoreOpener-7f47811997514991aba525c2f71bf84c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f47811997514991aba525c2f71bf84c columnFamilyName a 2024-11-12T10:30:05,549 DEBUG [StoreOpener-7f47811997514991aba525c2f71bf84c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:05,550 INFO [StoreOpener-7f47811997514991aba525c2f71bf84c-1 {}] regionserver.HStore(327): Store=7f47811997514991aba525c2f71bf84c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:05,551 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7f47811997514991aba525c2f71bf84c 2024-11-12T10:30:05,552 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c 2024-11-12T10:30:05,553 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c 2024-11-12T10:30:05,553 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7f47811997514991aba525c2f71bf84c 2024-11-12T10:30:05,553 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7f47811997514991aba525c2f71bf84c 2024-11-12T10:30:05,555 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7f47811997514991aba525c2f71bf84c 2024-11-12T10:30:05,559 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:30:05,559 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7f47811997514991aba525c2f71bf84c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63329500, jitterRate=-0.056316912174224854}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T10:30:05,560 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7f47811997514991aba525c2f71bf84c: Writing region info on filesystem at 1731407405544Initializing all the Stores at 1731407405546 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407405546Cleaning up temporary data from old regions at 1731407405553 (+7 ms)Region opened successfully at 1731407405560 (+7 ms) 2024-11-12T10:30:05,560 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7f47811997514991aba525c2f71bf84c, disabling compactions & flushes 2024-11-12T10:30:05,560 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731407405491.7f47811997514991aba525c2f71bf84c. 2024-11-12T10:30:05,560 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731407405491.7f47811997514991aba525c2f71bf84c. 2024-11-12T10:30:05,560 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731407405491.7f47811997514991aba525c2f71bf84c. after waiting 0 ms 2024-11-12T10:30:05,560 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731407405491.7f47811997514991aba525c2f71bf84c. 2024-11-12T10:30:05,560 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731407405491.7f47811997514991aba525c2f71bf84c. 2024-11-12T10:30:05,561 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7f47811997514991aba525c2f71bf84c: Waiting for close lock at 1731407405560Disabling compacts and flushes for region at 1731407405560Disabling writes for close at 1731407405560Writing region close event to WAL at 1731407405560Closed at 1731407405560 2024-11-12T10:30:05,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741845_1021 (size=93) 2024-11-12T10:30:05,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741845_1021 (size=93) 2024-11-12T10:30:05,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741845_1021 (size=93) 2024-11-12T10:30:05,577 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T10:30:05,577 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-45596041:(num 1731407405499) 2024-11-12T10:30:05,579 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-12T10:30:05,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741847_1023 (size=369) 2024-11-12T10:30:05,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741847_1023 (size=369) 2024-11-12T10:30:05,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741847_1023 (size=369) 2024-11-12T10:30:05,601 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-12T10:30:05,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741848_1024 (size=231) 2024-11-12T10:30:05,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741848_1024 (size=231) 2024-11-12T10:30:05,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741848_1024 (size=231) 2024-11-12T10:30:05,634 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-2, size=231 (231bytes) 2024-11-12T10:30:05,634 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-2 2024-11-12T10:30:05,635 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-2 after 1ms 2024-11-12T10:30:05,640 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-2 took 6ms 2024-11-12T10:30:05,643 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-2 so closing down 2024-11-12T10:30:05,644 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T10:30:05,646 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-12T10:30:05,648 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000002-wal-2.temp 2024-11-12T10:30:05,648 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T10:30:05,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741849_1025 (size=231) 2024-11-12T10:30:05,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741849_1025 (size=231) 2024-11-12T10:30:05,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741849_1025 (size=231) 2024-11-12T10:30:05,657 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-12T10:30:05,660 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000002 2024-11-12T10:30:05,660 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-12T10:30:05,660 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-2, journal: Splitting hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-2, size=231 (231bytes) at 1731407405634Finishing writing output for hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-2 so closing down at 1731407405643 (+9 ms)Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000002-wal-2.temp at 1731407405648 (+5 ms)3 split writer threads finished at 1731407405648Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731407405657 (+9 ms)Rename recovered edits hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000002 at 1731407405660 (+3 ms)Processed 1 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1731407405660 2024-11-12T10:30:05,680 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-1, size=369 (369bytes) 2024-11-12T10:30:05,680 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-1 2024-11-12T10:30:05,681 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-1 after 1ms 2024-11-12T10:30:05,685 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-1 took 5ms 2024-11-12T10:30:05,687 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-1 so closing down 2024-11-12T10:30:05,687 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T10:30:05,690 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-12T10:30:05,691 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000001-wal-1.temp 2024-11-12T10:30:05,692 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T10:30:05,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741850_1026 (size=369) 2024-11-12T10:30:05,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741850_1026 (size=369) 2024-11-12T10:30:05,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741850_1026 (size=369) 2024-11-12T10:30:05,699 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-12T10:30:05,706 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000002, length=231 2024-11-12T10:30:05,708 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000002 2024-11-12T10:30:05,708 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-12T10:30:05,709 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-1, journal: Splitting hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-1, size=369 (369bytes) at 1731407405680Finishing writing output for hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-1 so closing down at 1731407405687 (+7 ms)Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000001-wal-1.temp at 1731407405691 (+4 ms)3 split writer threads finished at 1731407405692 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731407405700 (+8 ms)Rename recovered edits hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000002 at 1731407405708 (+8 ms)Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1731407405708 2024-11-12T10:30:05,709 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:05,711 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:05,723 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal.1731407405712, exclude list is [], retry=0 2024-11-12T10:30:05,726 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:05,726 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:05,727 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:05,728 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal.1731407405712 2024-11-12T10:30:05,729 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:39823:39823)] 2024-11-12T10:30:05,729 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7f47811997514991aba525c2f71bf84c, NAME => 'testReplayEditsWrittenIntoWAL,,1731407405491.7f47811997514991aba525c2f71bf84c.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:05,729 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731407405491.7f47811997514991aba525c2f71bf84c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:05,729 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7f47811997514991aba525c2f71bf84c 2024-11-12T10:30:05,729 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7f47811997514991aba525c2f71bf84c 2024-11-12T10:30:05,731 INFO [StoreOpener-7f47811997514991aba525c2f71bf84c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7f47811997514991aba525c2f71bf84c 2024-11-12T10:30:05,733 INFO [StoreOpener-7f47811997514991aba525c2f71bf84c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f47811997514991aba525c2f71bf84c columnFamilyName a 2024-11-12T10:30:05,733 DEBUG [StoreOpener-7f47811997514991aba525c2f71bf84c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:05,734 INFO [StoreOpener-7f47811997514991aba525c2f71bf84c-1 {}] regionserver.HStore(327): Store=7f47811997514991aba525c2f71bf84c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:05,734 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7f47811997514991aba525c2f71bf84c 2024-11-12T10:30:05,735 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c 2024-11-12T10:30:05,737 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c 2024-11-12T10:30:05,738 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000002 2024-11-12T10:30:05,742 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000002 2024-11-12T10:30:05,742 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7f47811997514991aba525c2f71bf84c 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-12T10:30:05,757 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/.tmp/a/46701dd1c13e42209b4e6f38f3a94f60 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731407405577/Put/seqid=0 2024-11-12T10:30:05,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741852_1028 (size=5170) 2024-11-12T10:30:05,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741852_1028 (size=5170) 2024-11-12T10:30:05,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741852_1028 (size=5170) 2024-11-12T10:30:05,766 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/.tmp/a/46701dd1c13e42209b4e6f38f3a94f60 2024-11-12T10:30:05,775 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/.tmp/a/46701dd1c13e42209b4e6f38f3a94f60 as hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/a/46701dd1c13e42209b4e6f38f3a94f60 2024-11-12T10:30:05,784 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/a/46701dd1c13e42209b4e6f38f3a94f60, entries=2, sequenceid=2, filesize=5.0 K 2024-11-12T10:30:05,785 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 7f47811997514991aba525c2f71bf84c in 43ms, sequenceid=2, compaction requested=false; wal=null 2024-11-12T10:30:05,786 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/0000000000000000002 2024-11-12T10:30:05,786 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7f47811997514991aba525c2f71bf84c 2024-11-12T10:30:05,786 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7f47811997514991aba525c2f71bf84c 2024-11-12T10:30:05,789 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7f47811997514991aba525c2f71bf84c 2024-11-12T10:30:05,792 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/7f47811997514991aba525c2f71bf84c/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-12T10:30:05,793 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7f47811997514991aba525c2f71bf84c; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62257532, jitterRate=-0.07229048013687134}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T10:30:05,793 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7f47811997514991aba525c2f71bf84c: Writing region info on filesystem at 1731407405729Initializing all the Stores at 1731407405731 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407405731Obtaining lock to block concurrent updates at 1731407405742 (+11 ms)Preparing flush snapshotting stores in 7f47811997514991aba525c2f71bf84c at 1731407405742Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731407405491.7f47811997514991aba525c2f71bf84c., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731407405742Flushing stores of testReplayEditsWrittenIntoWAL,,1731407405491.7f47811997514991aba525c2f71bf84c. at 1731407405742Flushing 7f47811997514991aba525c2f71bf84c/a: creating writer at 1731407405742Flushing 7f47811997514991aba525c2f71bf84c/a: appending metadata at 1731407405755 (+13 ms)Flushing 7f47811997514991aba525c2f71bf84c/a: closing flushed file at 1731407405756 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50dceba4: reopening flushed file at 1731407405774 (+18 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 7f47811997514991aba525c2f71bf84c in 43ms, sequenceid=2, compaction requested=false; wal=null at 1731407405785 (+11 ms)Cleaning up temporary data from old regions at 1731407405786 (+1 ms)Region opened successfully at 1731407405793 (+7 ms) 2024-11-12T10:30:05,809 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=386 (was 376) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-663294313-172.17.0.3-1731407397077:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-663294313-172.17.0.3-1731407397077:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:35752 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:49404 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:49020 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:48838 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-663294313-172.17.0.3-1731407397077:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:49184 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:35638 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=849 (was 849), ProcessCount=11 (was 11), AvailableMemoryMB=7634 (was 7645) 2024-11-12T10:30:05,819 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=386, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=849, ProcessCount=11, AvailableMemoryMB=7633 2024-11-12T10:30:05,833 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:05,835 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:05,836 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T10:30:05,839 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-50733454, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/hregion-50733454, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:05,851 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-50733454/hregion-50733454.1731407405840, exclude list is [], retry=0 2024-11-12T10:30:05,855 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:05,855 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:05,856 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:05,858 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-50733454/hregion-50733454.1731407405840 2024-11-12T10:30:05,859 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33413:33413)] 2024-11-12T10:30:05,859 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f03fb419b4d1b2913d46e1b439cbfa07, NAME => 'testReplayEditsWrittenIntoWAL,,1731407405833.f03fb419b4d1b2913d46e1b439cbfa07.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38657/hbase 2024-11-12T10:30:05,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741854_1030 (size=64) 2024-11-12T10:30:05,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741854_1030 (size=64) 2024-11-12T10:30:05,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741854_1030 (size=64) 2024-11-12T10:30:05,874 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731407405833.f03fb419b4d1b2913d46e1b439cbfa07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:05,876 INFO [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f03fb419b4d1b2913d46e1b439cbfa07 2024-11-12T10:30:05,879 INFO [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f03fb419b4d1b2913d46e1b439cbfa07 columnFamilyName a 2024-11-12T10:30:05,879 DEBUG [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:05,880 INFO [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] regionserver.HStore(327): Store=f03fb419b4d1b2913d46e1b439cbfa07/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:05,880 INFO [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f03fb419b4d1b2913d46e1b439cbfa07 2024-11-12T10:30:05,881 INFO [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f03fb419b4d1b2913d46e1b439cbfa07 columnFamilyName b 2024-11-12T10:30:05,882 DEBUG [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:05,882 INFO [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] regionserver.HStore(327): Store=f03fb419b4d1b2913d46e1b439cbfa07/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:05,883 INFO [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f03fb419b4d1b2913d46e1b439cbfa07 2024-11-12T10:30:05,885 INFO [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f03fb419b4d1b2913d46e1b439cbfa07 columnFamilyName c 2024-11-12T10:30:05,885 DEBUG [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:05,886 INFO [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] regionserver.HStore(327): Store=f03fb419b4d1b2913d46e1b439cbfa07/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:05,886 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f03fb419b4d1b2913d46e1b439cbfa07 2024-11-12T10:30:05,887 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07 2024-11-12T10:30:05,887 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07 2024-11-12T10:30:05,889 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f03fb419b4d1b2913d46e1b439cbfa07 2024-11-12T10:30:05,889 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f03fb419b4d1b2913d46e1b439cbfa07 2024-11-12T10:30:05,890 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T10:30:05,891 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f03fb419b4d1b2913d46e1b439cbfa07 2024-11-12T10:30:05,893 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:30:05,894 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f03fb419b4d1b2913d46e1b439cbfa07; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60630038, jitterRate=-0.09654203057289124}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T10:30:05,894 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f03fb419b4d1b2913d46e1b439cbfa07: Writing region info on filesystem at 1731407405874Initializing all the Stores at 1731407405875 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407405875Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407405876 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407405876Cleaning up temporary data from old regions at 1731407405889 (+13 ms)Region opened successfully at 1731407405894 (+5 ms) 2024-11-12T10:30:05,894 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f03fb419b4d1b2913d46e1b439cbfa07, disabling compactions & flushes 2024-11-12T10:30:05,894 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731407405833.f03fb419b4d1b2913d46e1b439cbfa07. 2024-11-12T10:30:05,894 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731407405833.f03fb419b4d1b2913d46e1b439cbfa07. 2024-11-12T10:30:05,894 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731407405833.f03fb419b4d1b2913d46e1b439cbfa07. after waiting 0 ms 2024-11-12T10:30:05,895 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731407405833.f03fb419b4d1b2913d46e1b439cbfa07. 2024-11-12T10:30:05,895 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731407405833.f03fb419b4d1b2913d46e1b439cbfa07. 2024-11-12T10:30:05,895 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f03fb419b4d1b2913d46e1b439cbfa07: Waiting for close lock at 1731407405894Disabling compacts and flushes for region at 1731407405894Disabling writes for close at 1731407405894Writing region close event to WAL at 1731407405895 (+1 ms)Closed at 1731407405895 2024-11-12T10:30:05,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741853_1029 (size=93) 2024-11-12T10:30:05,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741853_1029 (size=93) 2024-11-12T10:30:05,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741853_1029 (size=93) 2024-11-12T10:30:05,902 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T10:30:05,902 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-50733454:(num 1731407405840) 2024-11-12T10:30:05,902 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:05,904 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:05,916 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407405905, exclude list is [], retry=0 2024-11-12T10:30:05,919 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:05,920 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:05,920 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:05,929 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407405905 2024-11-12T10:30:05,929 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33367:33367)] 2024-11-12T10:30:06,168 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407405905, size=0 (0bytes) 2024-11-12T10:30:06,168 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407405905 might be still open, length is 0 2024-11-12T10:30:06,168 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407405905 2024-11-12T10:30:06,169 WARN [IPC Server handler 4 on default port 38657 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407405905 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-12T10:30:06,170 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407405905 after 2ms 2024-11-12T10:30:06,731 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:35794 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:35741:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35794 dst: /127.0.0.1:35741 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35741 remote=/127.0.0.1:35794]. Total timeout mills is 60000, 59408 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:06,732 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:49444 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43905:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49444 dst: /127.0.0.1:43905 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:06,732 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:49040 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:34283:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49040 dst: /127.0.0.1:34283 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:06,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741855_1032 (size=470675) 2024-11-12T10:30:06,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741855_1032 (size=470675) 2024-11-12T10:30:06,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741855_1032 (size=470675) 2024-11-12T10:30:10,058 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T10:30:10,114 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T10:30:10,171 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407405905 after 4003ms 2024-11-12T10:30:10,178 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407405905 took 4010ms 2024-11-12T10:30:10,186 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731407405905.temp 2024-11-12T10:30:10,189 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/recovered.edits/0000000000000000001-wal.1731407405905.temp 2024-11-12T10:30:10,269 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407405905; continuing. 2024-11-12T10:30:10,269 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407405905 so closing down 2024-11-12T10:30:10,269 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T10:30:10,269 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T10:30:10,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741856_1033 (size=470683) 2024-11-12T10:30:10,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741856_1033 (size=470683) 2024-11-12T10:30:10,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741856_1033 (size=470683) 2024-11-12T10:30:10,273 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/recovered.edits/0000000000000000001-wal.1731407405905.temp (wrote 3002 edits, skipped 0 edits in 29 ms) 2024-11-12T10:30:10,275 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/recovered.edits/0000000000000000001-wal.1731407405905.temp to hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/recovered.edits/0000000000000003002 2024-11-12T10:30:10,275 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 96 ms; skipped=0; WAL=hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407405905, size=0, length=0, corrupted=false, cancelled=false 2024-11-12T10:30:10,275 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407405905, journal: Splitting hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407405905, size=0 (0bytes) at 1731407406168Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/recovered.edits/0000000000000000001-wal.1731407405905.temp at 1731407410189 (+4021 ms)Split 1024 edits, skipped 0 edits. at 1731407410234 (+45 ms)Split 2048 edits, skipped 0 edits. at 1731407410253 (+19 ms)Finishing writing output for hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407405905 so closing down at 1731407410269 (+16 ms)3 split writer threads finished at 1731407410269Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/recovered.edits/0000000000000000001-wal.1731407405905.temp (wrote 3002 edits, skipped 0 edits in 29 ms) at 1731407410273 (+4 ms)Rename recovered edits hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/recovered.edits/0000000000000000001-wal.1731407405905.temp to hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/recovered.edits/0000000000000003002 at 1731407410275 (+2 ms)Processed 3002 edits across 1 Regions in 96 ms; skipped=0; WAL=hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407405905, size=0, length=0, corrupted=false, cancelled=false at 1731407410275 2024-11-12T10:30:10,278 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407405905 to hdfs://localhost:38657/hbase/oldWALs/wal.1731407405905 2024-11-12T10:30:10,279 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/recovered.edits/0000000000000003002 2024-11-12T10:30:10,279 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:10,282 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:10,300 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407410283, exclude list is [], retry=0 2024-11-12T10:30:10,304 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:10,305 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:10,305 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:10,307 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407410283 2024-11-12T10:30:10,308 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:39823:39823)] 2024-11-12T10:30:10,309 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731407405833.f03fb419b4d1b2913d46e1b439cbfa07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:10,311 INFO [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f03fb419b4d1b2913d46e1b439cbfa07 2024-11-12T10:30:10,312 INFO [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f03fb419b4d1b2913d46e1b439cbfa07 columnFamilyName a 2024-11-12T10:30:10,312 DEBUG [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:10,312 INFO [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] regionserver.HStore(327): Store=f03fb419b4d1b2913d46e1b439cbfa07/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:10,313 INFO [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f03fb419b4d1b2913d46e1b439cbfa07 2024-11-12T10:30:10,314 INFO [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f03fb419b4d1b2913d46e1b439cbfa07 columnFamilyName b 2024-11-12T10:30:10,314 DEBUG [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:10,314 INFO [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] regionserver.HStore(327): Store=f03fb419b4d1b2913d46e1b439cbfa07/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:10,314 INFO [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f03fb419b4d1b2913d46e1b439cbfa07 2024-11-12T10:30:10,315 INFO [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f03fb419b4d1b2913d46e1b439cbfa07 columnFamilyName c 2024-11-12T10:30:10,315 DEBUG [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:10,316 INFO [StoreOpener-f03fb419b4d1b2913d46e1b439cbfa07-1 {}] regionserver.HStore(327): Store=f03fb419b4d1b2913d46e1b439cbfa07/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:10,316 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f03fb419b4d1b2913d46e1b439cbfa07 2024-11-12T10:30:10,317 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07 2024-11-12T10:30:10,319 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07 2024-11-12T10:30:10,320 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/recovered.edits/0000000000000003002 2024-11-12T10:30:10,359 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-12T10:30:10,703 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f03fb419b4d1b2913d46e1b439cbfa07 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-12T10:30:10,739 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/.tmp/a/ce2c18b9216f4a6899a4a4ccacd8fd9b is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731407405941/Put/seqid=0 2024-11-12T10:30:10,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741858_1035 (size=50463) 2024-11-12T10:30:10,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741858_1035 (size=50463) 2024-11-12T10:30:10,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741858_1035 (size=50463) 2024-11-12T10:30:10,747 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/.tmp/a/ce2c18b9216f4a6899a4a4ccacd8fd9b 2024-11-12T10:30:10,755 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/.tmp/a/ce2c18b9216f4a6899a4a4ccacd8fd9b as hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/a/ce2c18b9216f4a6899a4a4ccacd8fd9b 2024-11-12T10:30:10,762 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/a/ce2c18b9216f4a6899a4a4ccacd8fd9b, entries=754, sequenceid=754, filesize=49.3 K 2024-11-12T10:30:10,763 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for f03fb419b4d1b2913d46e1b439cbfa07 in 60ms, sequenceid=754, compaction requested=false; wal=null 2024-11-12T10:30:10,783 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-12T10:30:10,783 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f03fb419b4d1b2913d46e1b439cbfa07 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-12T10:30:10,793 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/.tmp/a/d7498a1f740848d4a31cc55778e08922 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731407405997/Put/seqid=0 2024-11-12T10:30:10,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741859_1036 (size=20072) 2024-11-12T10:30:10,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741859_1036 (size=20072) 2024-11-12T10:30:10,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741859_1036 (size=20072) 2024-11-12T10:30:10,806 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/.tmp/a/d7498a1f740848d4a31cc55778e08922 2024-11-12T10:30:10,830 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/.tmp/b/62fe6f6f140a4b1d827a91bf8c138e5a is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731407406020/Put/seqid=0 2024-11-12T10:30:10,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741860_1037 (size=35835) 2024-11-12T10:30:10,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741860_1037 (size=35835) 2024-11-12T10:30:10,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741860_1037 (size=35835) 2024-11-12T10:30:10,839 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/.tmp/b/62fe6f6f140a4b1d827a91bf8c138e5a 2024-11-12T10:30:10,848 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/.tmp/a/d7498a1f740848d4a31cc55778e08922 as hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/a/d7498a1f740848d4a31cc55778e08922 2024-11-12T10:30:10,855 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/a/d7498a1f740848d4a31cc55778e08922, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-12T10:30:10,856 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/.tmp/b/62fe6f6f140a4b1d827a91bf8c138e5a as hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/b/62fe6f6f140a4b1d827a91bf8c138e5a 2024-11-12T10:30:10,863 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/b/62fe6f6f140a4b1d827a91bf8c138e5a, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-12T10:30:10,864 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for f03fb419b4d1b2913d46e1b439cbfa07 in 80ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-12T10:30:10,874 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-12T10:30:10,874 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f03fb419b4d1b2913d46e1b439cbfa07 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-12T10:30:10,882 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/.tmp/b/41f83b3478224b34a9dc2ce0d298759b is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731407406045/Put/seqid=0 2024-11-12T10:30:10,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741861_1038 (size=35082) 2024-11-12T10:30:10,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741861_1038 (size=35082) 2024-11-12T10:30:10,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741861_1038 (size=35082) 2024-11-12T10:30:10,891 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/.tmp/b/41f83b3478224b34a9dc2ce0d298759b 2024-11-12T10:30:10,918 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/.tmp/c/3131e023f04f4393a3aa68e93b99fe0a is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731407406082/Put/seqid=0 2024-11-12T10:30:10,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741862_1039 (size=20825) 2024-11-12T10:30:10,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741862_1039 (size=20825) 2024-11-12T10:30:10,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741862_1039 (size=20825) 2024-11-12T10:30:10,927 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/.tmp/c/3131e023f04f4393a3aa68e93b99fe0a 2024-11-12T10:30:10,936 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/.tmp/b/41f83b3478224b34a9dc2ce0d298759b as hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/b/41f83b3478224b34a9dc2ce0d298759b 2024-11-12T10:30:10,945 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/b/41f83b3478224b34a9dc2ce0d298759b, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-12T10:30:10,947 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/.tmp/c/3131e023f04f4393a3aa68e93b99fe0a as hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/c/3131e023f04f4393a3aa68e93b99fe0a 2024-11-12T10:30:10,955 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/c/3131e023f04f4393a3aa68e93b99fe0a, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-12T10:30:10,955 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for f03fb419b4d1b2913d46e1b439cbfa07 in 81ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-12T10:30:10,963 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731407406137/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731407405833.f03fb419b4d1b2913d46e1b439cbfa07. 2024-11-12T10:30:10,966 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/recovered.edits/0000000000000003002 2024-11-12T10:30:10,966 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-12T10:30:10,967 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f03fb419b4d1b2913d46e1b439cbfa07 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-12T10:30:10,975 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/.tmp/c/1e9a4288bcaf4444809d5379ee2d1cb2 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731407406089/Put/seqid=0 2024-11-12T10:30:10,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741863_1040 (size=50301) 2024-11-12T10:30:10,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741863_1040 (size=50301) 2024-11-12T10:30:10,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741863_1040 (size=50301) 2024-11-12T10:30:10,983 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/.tmp/c/1e9a4288bcaf4444809d5379ee2d1cb2 2024-11-12T10:30:10,990 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1e9a4288bcaf4444809d5379ee2d1cb2 2024-11-12T10:30:10,992 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/.tmp/c/1e9a4288bcaf4444809d5379ee2d1cb2 as hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/c/1e9a4288bcaf4444809d5379ee2d1cb2 2024-11-12T10:30:10,998 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1e9a4288bcaf4444809d5379ee2d1cb2 2024-11-12T10:30:10,998 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/c/1e9a4288bcaf4444809d5379ee2d1cb2, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-12T10:30:10,999 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for f03fb419b4d1b2913d46e1b439cbfa07 in 33ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-12T10:30:11,000 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/recovered.edits/0000000000000003002 2024-11-12T10:30:11,001 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f03fb419b4d1b2913d46e1b439cbfa07 2024-11-12T10:30:11,001 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f03fb419b4d1b2913d46e1b439cbfa07 2024-11-12T10:30:11,002 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T10:30:11,004 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f03fb419b4d1b2913d46e1b439cbfa07 2024-11-12T10:30:11,007 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenIntoWAL/f03fb419b4d1b2913d46e1b439cbfa07/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-12T10:30:11,008 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f03fb419b4d1b2913d46e1b439cbfa07; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67705600, jitterRate=0.008892059326171875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T10:30:11,008 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f03fb419b4d1b2913d46e1b439cbfa07: Writing region info on filesystem at 1731407410309Initializing all the Stores at 1731407410310 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407410310Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407410310Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407410311 (+1 ms)Cleaning up temporary data from old regions at 1731407411001 (+690 ms)Region opened successfully at 1731407411008 (+7 ms) 2024-11-12T10:30:11,071 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f03fb419b4d1b2913d46e1b439cbfa07, disabling compactions & flushes 2024-11-12T10:30:11,071 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731407405833.f03fb419b4d1b2913d46e1b439cbfa07. 2024-11-12T10:30:11,071 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731407405833.f03fb419b4d1b2913d46e1b439cbfa07. 2024-11-12T10:30:11,071 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731407405833.f03fb419b4d1b2913d46e1b439cbfa07. after waiting 0 ms 2024-11-12T10:30:11,071 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731407405833.f03fb419b4d1b2913d46e1b439cbfa07. 2024-11-12T10:30:11,073 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731407405833.f03fb419b4d1b2913d46e1b439cbfa07. 2024-11-12T10:30:11,073 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f03fb419b4d1b2913d46e1b439cbfa07: Waiting for close lock at 1731407411071Disabling compacts and flushes for region at 1731407411071Disabling writes for close at 1731407411071Writing region close event to WAL at 1731407411073 (+2 ms)Closed at 1731407411073 2024-11-12T10:30:11,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741857_1034 (size=93) 2024-11-12T10:30:11,077 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407410283 not finished, retry = 0 2024-11-12T10:30:11,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741857_1034 (size=93) 2024-11-12T10:30:11,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741857_1034 (size=93) 2024-11-12T10:30:11,180 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T10:30:11,181 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731407410283) 2024-11-12T10:30:11,193 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=400 (was 386) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:38657 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Client (865318088) connection to localhost/127.0.0.1:46857 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46857 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45873 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1412168931_22 at /127.0.0.1:49066 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (865318088) connection to localhost/127.0.0.1:38657 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1412168931_22 at /127.0.0.1:35818 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1412168931_22 at /127.0.0.1:49462 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1412168931_22 at /127.0.0.1:49094 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (865318088) connection to localhost/127.0.0.1:45873 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=781 (was 849), ProcessCount=11 (was 11), AvailableMemoryMB=7617 (was 7633) 2024-11-12T10:30:11,206 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=400, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=781, ProcessCount=11, AvailableMemoryMB=7616 2024-11-12T10:30:11,223 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:11,225 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:11,226 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T10:30:11,229 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-80090414, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/hregion-80090414, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:11,241 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-80090414/hregion-80090414.1731407411230, exclude list is [], retry=0 2024-11-12T10:30:11,244 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:11,244 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:11,244 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:11,248 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-80090414/hregion-80090414.1731407411230 2024-11-12T10:30:11,248 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:33413:33413)] 2024-11-12T10:30:11,249 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 089c0cbf819d00fa4fa637bf6ab5a890, NAME => 'test2727,,1731407411224.089c0cbf819d00fa4fa637bf6ab5a890.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38657/hbase 2024-11-12T10:30:11,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741865_1042 (size=43) 2024-11-12T10:30:11,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741865_1042 (size=43) 2024-11-12T10:30:11,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741865_1042 (size=43) 2024-11-12T10:30:11,262 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731407411224.089c0cbf819d00fa4fa637bf6ab5a890.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:11,263 INFO [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:11,265 INFO [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 089c0cbf819d00fa4fa637bf6ab5a890 columnFamilyName a 2024-11-12T10:30:11,265 DEBUG [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:11,266 INFO [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] regionserver.HStore(327): Store=089c0cbf819d00fa4fa637bf6ab5a890/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:11,266 INFO [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:11,267 INFO [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 089c0cbf819d00fa4fa637bf6ab5a890 columnFamilyName b 2024-11-12T10:30:11,267 DEBUG [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:11,268 INFO [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] regionserver.HStore(327): Store=089c0cbf819d00fa4fa637bf6ab5a890/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:11,268 INFO [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:11,270 INFO [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 089c0cbf819d00fa4fa637bf6ab5a890 columnFamilyName c 2024-11-12T10:30:11,270 DEBUG [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:11,271 INFO [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] regionserver.HStore(327): Store=089c0cbf819d00fa4fa637bf6ab5a890/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:11,271 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:11,272 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:11,272 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:11,273 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:11,273 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:11,274 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T10:30:11,275 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:11,278 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:30:11,278 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 089c0cbf819d00fa4fa637bf6ab5a890; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62446807, jitterRate=-0.06947006285190582}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T10:30:11,279 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 089c0cbf819d00fa4fa637bf6ab5a890: Writing region info on filesystem at 1731407411262Initializing all the Stores at 1731407411263 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407411263Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407411263Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407411263Cleaning up temporary data from old regions at 1731407411273 (+10 ms)Region opened successfully at 1731407411279 (+6 ms) 2024-11-12T10:30:11,279 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 089c0cbf819d00fa4fa637bf6ab5a890, disabling compactions & flushes 2024-11-12T10:30:11,279 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731407411224.089c0cbf819d00fa4fa637bf6ab5a890. 2024-11-12T10:30:11,279 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731407411224.089c0cbf819d00fa4fa637bf6ab5a890. 2024-11-12T10:30:11,279 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731407411224.089c0cbf819d00fa4fa637bf6ab5a890. after waiting 0 ms 2024-11-12T10:30:11,279 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731407411224.089c0cbf819d00fa4fa637bf6ab5a890. 2024-11-12T10:30:11,280 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731407411224.089c0cbf819d00fa4fa637bf6ab5a890. 2024-11-12T10:30:11,280 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 089c0cbf819d00fa4fa637bf6ab5a890: Waiting for close lock at 1731407411279Disabling compacts and flushes for region at 1731407411279Disabling writes for close at 1731407411279Writing region close event to WAL at 1731407411280 (+1 ms)Closed at 1731407411280 2024-11-12T10:30:11,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741864_1041 (size=93) 2024-11-12T10:30:11,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741864_1041 (size=93) 2024-11-12T10:30:11,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741864_1041 (size=93) 2024-11-12T10:30:11,286 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T10:30:11,286 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-80090414:(num 1731407411230) 2024-11-12T10:30:11,286 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:11,288 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:11,300 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411289, exclude list is [], retry=0 2024-11-12T10:30:11,303 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:11,304 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:11,304 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:11,308 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411289 2024-11-12T10:30:11,309 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:39823:39823)] 2024-11-12T10:30:11,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741866_1043 (size=344306) 2024-11-12T10:30:11,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741866_1043 (size=344306) 2024-11-12T10:30:11,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741866_1043 (size=344306) 2024-11-12T10:30:11,494 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411289, size=336.2 K (344306bytes) 2024-11-12T10:30:11,494 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411289 2024-11-12T10:30:11,495 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411289 after 1ms 2024-11-12T10:30:11,499 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411289 took 5ms 2024-11-12T10:30:11,506 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731407411289.temp 2024-11-12T10:30:11,509 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000000001-wal.1731407411289.temp 2024-11-12T10:30:11,527 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411289 so closing down 2024-11-12T10:30:11,527 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T10:30:11,528 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T10:30:11,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741867_1044 (size=344306) 2024-11-12T10:30:11,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741867_1044 (size=344306) 2024-11-12T10:30:11,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741867_1044 (size=344306) 2024-11-12T10:30:11,535 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000000001-wal.1731407411289.temp (wrote 3000 edits, skipped 0 edits in 8 ms) 2024-11-12T10:30:11,537 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000000001-wal.1731407411289.temp to hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000003000 2024-11-12T10:30:11,537 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 38 ms; skipped=0; WAL=hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411289, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-11-12T10:30:11,537 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411289, journal: Splitting hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411289, size=336.2 K (344306bytes) at 1731407411494Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000000001-wal.1731407411289.temp at 1731407411509 (+15 ms)Split 1024 edits, skipped 0 edits. at 1731407411510 (+1 ms)Split 2048 edits, skipped 0 edits. at 1731407411518 (+8 ms)Finishing writing output for hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411289 so closing down at 1731407411527 (+9 ms)3 split writer threads finished at 1731407411528 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000000001-wal.1731407411289.temp (wrote 3000 edits, skipped 0 edits in 8 ms) at 1731407411535 (+7 ms)Rename recovered edits hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000000001-wal.1731407411289.temp to hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000003000 at 1731407411537 (+2 ms)Processed 3000 edits across 1 Regions in 38 ms; skipped=0; WAL=hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411289, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1731407411537 2024-11-12T10:30:11,540 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411289 to hdfs://localhost:38657/hbase/oldWALs/wal.1731407411289 2024-11-12T10:30:11,541 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000003000 2024-11-12T10:30:11,541 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:11,544 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:11,563 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411545, exclude list is [], retry=0 2024-11-12T10:30:11,566 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:11,567 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:11,567 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:11,571 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411545 2024-11-12T10:30:11,571 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33413:33413)] 2024-11-12T10:30:11,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741868_1045 (size=344433) 2024-11-12T10:30:11,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741868_1045 (size=344433) 2024-11-12T10:30:11,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741868_1045 (size=344433) 2024-11-12T10:30:11,756 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411545, size=336.4 K (344433bytes) 2024-11-12T10:30:11,756 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411545 2024-11-12T10:30:11,757 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411545 after 1ms 2024-11-12T10:30:11,762 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411545 took 6ms 2024-11-12T10:30:11,766 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731407411545.temp 2024-11-12T10:30:11,768 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000003001-wal.1731407411545.temp 2024-11-12T10:30:11,795 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411545 so closing down 2024-11-12T10:30:11,796 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T10:30:11,796 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T10:30:11,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741869_1046 (size=344433) 2024-11-12T10:30:11,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741869_1046 (size=344433) 2024-11-12T10:30:11,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741869_1046 (size=344433) 2024-11-12T10:30:11,803 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000003001-wal.1731407411545.temp (wrote 3000 edits, skipped 0 edits in 10 ms) 2024-11-12T10:30:11,805 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000003001-wal.1731407411545.temp to hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000006000 2024-11-12T10:30:11,805 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 43 ms; skipped=0; WAL=hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411545, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-11-12T10:30:11,806 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411545, journal: Splitting hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411545, size=336.4 K (344433bytes) at 1731407411756Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000003001-wal.1731407411545.temp at 1731407411768 (+12 ms)Split 1024 edits, skipped 0 edits. at 1731407411774 (+6 ms)Split 2048 edits, skipped 0 edits. at 1731407411785 (+11 ms)Finishing writing output for hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411545 so closing down at 1731407411796 (+11 ms)3 split writer threads finished at 1731407411796Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000003001-wal.1731407411545.temp (wrote 3000 edits, skipped 0 edits in 10 ms) at 1731407411803 (+7 ms)Rename recovered edits hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000003001-wal.1731407411545.temp to hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000006000 at 1731407411805 (+2 ms)Processed 3000 edits across 1 Regions in 43 ms; skipped=0; WAL=hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411545, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1731407411805 2024-11-12T10:30:11,808 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411545 to hdfs://localhost:38657/hbase/oldWALs/wal.1731407411545 2024-11-12T10:30:11,809 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000006000 2024-11-12T10:30:11,809 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:11,812 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/test2727-manual,16010,1731407411222, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:11,833 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411813, exclude list is [], retry=0 2024-11-12T10:30:11,836 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:11,837 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:11,837 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:11,839 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731407411222/wal.1731407411813 2024-11-12T10:30:11,839 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33413:33413)] 2024-11-12T10:30:11,840 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 089c0cbf819d00fa4fa637bf6ab5a890, NAME => 'test2727,,1731407411224.089c0cbf819d00fa4fa637bf6ab5a890.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:11,840 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731407411224.089c0cbf819d00fa4fa637bf6ab5a890.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:11,840 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:11,840 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:11,842 INFO [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:11,843 INFO [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 089c0cbf819d00fa4fa637bf6ab5a890 columnFamilyName a 2024-11-12T10:30:11,844 DEBUG [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:11,844 INFO [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] regionserver.HStore(327): Store=089c0cbf819d00fa4fa637bf6ab5a890/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:11,844 INFO [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:11,846 INFO [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 089c0cbf819d00fa4fa637bf6ab5a890 columnFamilyName b 2024-11-12T10:30:11,846 DEBUG [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:11,847 INFO [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] regionserver.HStore(327): Store=089c0cbf819d00fa4fa637bf6ab5a890/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:11,847 INFO [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:11,848 INFO [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 089c0cbf819d00fa4fa637bf6ab5a890 columnFamilyName c 2024-11-12T10:30:11,848 DEBUG [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:11,849 INFO [StoreOpener-089c0cbf819d00fa4fa637bf6ab5a890-1 {}] regionserver.HStore(327): Store=089c0cbf819d00fa4fa637bf6ab5a890/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:11,849 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:11,850 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:11,853 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:11,854 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000003000 2024-11-12T10:30:11,899 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000003000 2024-11-12T10:30:11,901 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000006000 2024-11-12T10:30:11,948 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000006000 2024-11-12T10:30:11,948 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 089c0cbf819d00fa4fa637bf6ab5a890 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-12T10:30:11,968 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/.tmp/a/f44b8950497f48f5bfd17659eb0b445b is 41, key is test2727/a:100/1731407411578/Put/seqid=0 2024-11-12T10:30:11,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741871_1048 (size=84227) 2024-11-12T10:30:11,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741871_1048 (size=84227) 2024-11-12T10:30:11,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741871_1048 (size=84227) 2024-11-12T10:30:11,976 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/.tmp/a/f44b8950497f48f5bfd17659eb0b445b 2024-11-12T10:30:12,005 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/.tmp/b/f6a3626d955542bd8b793f4fd3fb793c is 41, key is test2727/b:100/1731407411642/Put/seqid=0 2024-11-12T10:30:12,006 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-12T10:30:12,006 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-12T10:30:12,008 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T10:30:12,008 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-12T10:30:12,008 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-12T10:30:12,008 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-12T10:30:12,009 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-12T10:30:12,009 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-12T10:30:12,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741872_1049 (size=84609) 2024-11-12T10:30:12,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741872_1049 (size=84609) 2024-11-12T10:30:12,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741872_1049 (size=84609) 2024-11-12T10:30:12,014 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/.tmp/b/f6a3626d955542bd8b793f4fd3fb793c 2024-11-12T10:30:12,050 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/.tmp/c/157d77aff8d6488bbec8435722e75f42 is 41, key is test2727/c:100/1731407411700/Put/seqid=0 2024-11-12T10:30:12,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741873_1050 (size=84609) 2024-11-12T10:30:12,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741873_1050 (size=84609) 2024-11-12T10:30:12,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741873_1050 (size=84609) 2024-11-12T10:30:12,057 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/.tmp/c/157d77aff8d6488bbec8435722e75f42 2024-11-12T10:30:12,064 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/.tmp/a/f44b8950497f48f5bfd17659eb0b445b as hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/a/f44b8950497f48f5bfd17659eb0b445b 2024-11-12T10:30:12,071 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/a/f44b8950497f48f5bfd17659eb0b445b, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-12T10:30:12,073 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/.tmp/b/f6a3626d955542bd8b793f4fd3fb793c as hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/b/f6a3626d955542bd8b793f4fd3fb793c 2024-11-12T10:30:12,080 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/b/f6a3626d955542bd8b793f4fd3fb793c, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-12T10:30:12,082 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/.tmp/c/157d77aff8d6488bbec8435722e75f42 as hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/c/157d77aff8d6488bbec8435722e75f42 2024-11-12T10:30:12,089 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/c/157d77aff8d6488bbec8435722e75f42, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-12T10:30:12,089 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 089c0cbf819d00fa4fa637bf6ab5a890 in 141ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-12T10:30:12,090 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000003000 2024-11-12T10:30:12,091 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/0000000000000006000 2024-11-12T10:30:12,092 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:12,092 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:12,093 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T10:30:12,095 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 089c0cbf819d00fa4fa637bf6ab5a890 2024-11-12T10:30:12,098 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/test2727/089c0cbf819d00fa4fa637bf6ab5a890/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-12T10:30:12,099 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 089c0cbf819d00fa4fa637bf6ab5a890; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67019231, jitterRate=-0.001335635781288147}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T10:30:12,101 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 089c0cbf819d00fa4fa637bf6ab5a890: Writing region info on filesystem at 1731407411840Initializing all the Stores at 1731407411841 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407411841Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407411842 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407411842Obtaining lock to block concurrent updates at 1731407411948 (+106 ms)Preparing flush snapshotting stores in 089c0cbf819d00fa4fa637bf6ab5a890 at 1731407411948Finished memstore snapshotting test2727,,1731407411224.089c0cbf819d00fa4fa637bf6ab5a890., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731407411948Flushing stores of test2727,,1731407411224.089c0cbf819d00fa4fa637bf6ab5a890. at 1731407411948Flushing 089c0cbf819d00fa4fa637bf6ab5a890/a: creating writer at 1731407411948Flushing 089c0cbf819d00fa4fa637bf6ab5a890/a: appending metadata at 1731407411968 (+20 ms)Flushing 089c0cbf819d00fa4fa637bf6ab5a890/a: closing flushed file at 1731407411968Flushing 089c0cbf819d00fa4fa637bf6ab5a890/b: creating writer at 1731407411983 (+15 ms)Flushing 089c0cbf819d00fa4fa637bf6ab5a890/b: appending metadata at 1731407412003 (+20 ms)Flushing 089c0cbf819d00fa4fa637bf6ab5a890/b: closing flushed file at 1731407412003Flushing 089c0cbf819d00fa4fa637bf6ab5a890/c: creating writer at 1731407412022 (+19 ms)Flushing 089c0cbf819d00fa4fa637bf6ab5a890/c: appending metadata at 1731407412049 (+27 ms)Flushing 089c0cbf819d00fa4fa637bf6ab5a890/c: closing flushed file at 1731407412050 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2532d956: reopening flushed file at 1731407412063 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@283c368b: reopening flushed file at 1731407412072 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ef4c4fc: reopening flushed file at 1731407412080 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 089c0cbf819d00fa4fa637bf6ab5a890 in 141ms, sequenceid=6000, compaction requested=false; wal=null at 1731407412089 (+9 ms)Cleaning up temporary data from old regions at 1731407412092 (+3 ms)Region opened successfully at 1731407412100 (+8 ms) 2024-11-12T10:30:12,101 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-12T10:30:12,102 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 089c0cbf819d00fa4fa637bf6ab5a890, disabling compactions & flushes 2024-11-12T10:30:12,102 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731407411224.089c0cbf819d00fa4fa637bf6ab5a890. 2024-11-12T10:30:12,102 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731407411224.089c0cbf819d00fa4fa637bf6ab5a890. 2024-11-12T10:30:12,102 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731407411224.089c0cbf819d00fa4fa637bf6ab5a890. after waiting 0 ms 2024-11-12T10:30:12,102 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731407411224.089c0cbf819d00fa4fa637bf6ab5a890. 2024-11-12T10:30:12,103 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731407411224.089c0cbf819d00fa4fa637bf6ab5a890. 2024-11-12T10:30:12,103 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 089c0cbf819d00fa4fa637bf6ab5a890: Waiting for close lock at 1731407412102Disabling compacts and flushes for region at 1731407412102Disabling writes for close at 1731407412102Writing region close event to WAL at 1731407412103 (+1 ms)Closed at 1731407412103 2024-11-12T10:30:12,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741870_1047 (size=93) 2024-11-12T10:30:12,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741870_1047 (size=93) 2024-11-12T10:30:12,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741870_1047 (size=93) 2024-11-12T10:30:12,109 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T10:30:12,109 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731407411813) 2024-11-12T10:30:12,121 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=402 (was 400) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:49640 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:35818 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:46624 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=781 (was 781), ProcessCount=11 (was 11), AvailableMemoryMB=7502 (was 7616) 2024-11-12T10:30:12,133 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=402, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=781, ProcessCount=11, AvailableMemoryMB=7501 2024-11-12T10:30:12,150 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:12,157 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731407412150, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:12,157 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731407412157 2024-11-12T10:30:12,164 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731407412150/wal.1731407412157 2024-11-12T10:30:12,168 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:39823:39823)] 2024-11-12T10:30:12,171 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d7adfde8f7246cc0322536ae9f0d55a, NAME => 'testSequentialEditLogSeqNum,,1731407412151.1d7adfde8f7246cc0322536ae9f0d55a.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:12,171 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731407412151.1d7adfde8f7246cc0322536ae9f0d55a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:12,171 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1d7adfde8f7246cc0322536ae9f0d55a 2024-11-12T10:30:12,171 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1d7adfde8f7246cc0322536ae9f0d55a 2024-11-12T10:30:12,172 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:38657/hbase/data/default/testSequentialEditLogSeqNum/1d7adfde8f7246cc0322536ae9f0d55a doesn't exist for region: 1d7adfde8f7246cc0322536ae9f0d55a on table testSequentialEditLogSeqNum 2024-11-12T10:30:12,173 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 1d7adfde8f7246cc0322536ae9f0d55a on table testSequentialEditLogSeqNum 2024-11-12T10:30:12,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741875_1052 (size=62) 2024-11-12T10:30:12,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741875_1052 (size=62) 2024-11-12T10:30:12,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741875_1052 (size=62) 2024-11-12T10:30:12,183 INFO [StoreOpener-1d7adfde8f7246cc0322536ae9f0d55a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1d7adfde8f7246cc0322536ae9f0d55a 2024-11-12T10:30:12,184 INFO [StoreOpener-1d7adfde8f7246cc0322536ae9f0d55a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d7adfde8f7246cc0322536ae9f0d55a columnFamilyName a 2024-11-12T10:30:12,184 DEBUG [StoreOpener-1d7adfde8f7246cc0322536ae9f0d55a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:12,185 INFO [StoreOpener-1d7adfde8f7246cc0322536ae9f0d55a-1 {}] regionserver.HStore(327): Store=1d7adfde8f7246cc0322536ae9f0d55a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:12,185 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1d7adfde8f7246cc0322536ae9f0d55a 2024-11-12T10:30:12,186 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testSequentialEditLogSeqNum/1d7adfde8f7246cc0322536ae9f0d55a 2024-11-12T10:30:12,186 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testSequentialEditLogSeqNum/1d7adfde8f7246cc0322536ae9f0d55a 2024-11-12T10:30:12,186 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1d7adfde8f7246cc0322536ae9f0d55a 2024-11-12T10:30:12,186 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1d7adfde8f7246cc0322536ae9f0d55a 2024-11-12T10:30:12,188 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1d7adfde8f7246cc0322536ae9f0d55a 2024-11-12T10:30:12,191 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testSequentialEditLogSeqNum/1d7adfde8f7246cc0322536ae9f0d55a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:30:12,192 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1d7adfde8f7246cc0322536ae9f0d55a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60079455, jitterRate=-0.10474635660648346}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T10:30:12,193 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1d7adfde8f7246cc0322536ae9f0d55a: Writing region info on filesystem at 1731407412171Initializing all the Stores at 1731407412182 (+11 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407412182Cleaning up temporary data from old regions at 1731407412186 (+4 ms)Region opened successfully at 1731407412193 (+7 ms) 2024-11-12T10:30:12,204 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1d7adfde8f7246cc0322536ae9f0d55a 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-12T10:30:12,224 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testSequentialEditLogSeqNum/1d7adfde8f7246cc0322536ae9f0d55a/.tmp/a/4e465d0a730740e3882625c028079a1d is 81, key is testSequentialEditLogSeqNum/a:x0/1731407412193/Put/seqid=0 2024-11-12T10:30:12,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741876_1053 (size=5833) 2024-11-12T10:30:12,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741876_1053 (size=5833) 2024-11-12T10:30:12,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741876_1053 (size=5833) 2024-11-12T10:30:12,232 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testSequentialEditLogSeqNum/1d7adfde8f7246cc0322536ae9f0d55a/.tmp/a/4e465d0a730740e3882625c028079a1d 2024-11-12T10:30:12,239 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testSequentialEditLogSeqNum/1d7adfde8f7246cc0322536ae9f0d55a/.tmp/a/4e465d0a730740e3882625c028079a1d as hdfs://localhost:38657/hbase/data/default/testSequentialEditLogSeqNum/1d7adfde8f7246cc0322536ae9f0d55a/a/4e465d0a730740e3882625c028079a1d 2024-11-12T10:30:12,247 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testSequentialEditLogSeqNum/1d7adfde8f7246cc0322536ae9f0d55a/a/4e465d0a730740e3882625c028079a1d, entries=10, sequenceid=13, filesize=5.7 K 2024-11-12T10:30:12,249 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 1d7adfde8f7246cc0322536ae9f0d55a in 44ms, sequenceid=13, compaction requested=false 2024-11-12T10:30:12,249 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1d7adfde8f7246cc0322536ae9f0d55a: 2024-11-12T10:30:12,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T10:30:12,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T10:30:12,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T10:30:12,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T10:30:12,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-12T10:30:12,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741874_1051 (size=3017) 2024-11-12T10:30:12,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741874_1051 (size=3017) 2024-11-12T10:30:12,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741874_1051 (size=3017) 2024-11-12T10:30:12,275 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731407412150/wal.1731407412157, size=2.9 K (3017bytes) 2024-11-12T10:30:12,275 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731407412150/wal.1731407412157 2024-11-12T10:30:12,276 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731407412150/wal.1731407412157 after 1ms 2024-11-12T10:30:12,279 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731407412150/wal.1731407412157 took 4ms 2024-11-12T10:30:12,282 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731407412150/wal.1731407412157 so closing down 2024-11-12T10:30:12,282 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T10:30:12,283 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731407412157.temp 2024-11-12T10:30:12,285 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testSequentialEditLogSeqNum/1d7adfde8f7246cc0322536ae9f0d55a/recovered.edits/0000000000000000003-wal.1731407412157.temp 2024-11-12T10:30:12,285 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T10:30:12,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741877_1054 (size=2433) 2024-11-12T10:30:12,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741877_1054 (size=2433) 2024-11-12T10:30:12,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741877_1054 (size=2433) 2024-11-12T10:30:12,293 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testSequentialEditLogSeqNum/1d7adfde8f7246cc0322536ae9f0d55a/recovered.edits/0000000000000000003-wal.1731407412157.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-12T10:30:12,295 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38657/hbase/data/default/testSequentialEditLogSeqNum/1d7adfde8f7246cc0322536ae9f0d55a/recovered.edits/0000000000000000003-wal.1731407412157.temp to hdfs://localhost:38657/hbase/data/default/testSequentialEditLogSeqNum/1d7adfde8f7246cc0322536ae9f0d55a/recovered.edits/0000000000000000020 2024-11-12T10:30:12,295 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:38657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731407412150/wal.1731407412157, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-11-12T10:30:12,296 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731407412150/wal.1731407412157, journal: Splitting hdfs://localhost:38657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731407412150/wal.1731407412157, size=2.9 K (3017bytes) at 1731407412275Finishing writing output for hdfs://localhost:38657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731407412150/wal.1731407412157 so closing down at 1731407412282 (+7 ms)Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testSequentialEditLogSeqNum/1d7adfde8f7246cc0322536ae9f0d55a/recovered.edits/0000000000000000003-wal.1731407412157.temp at 1731407412285 (+3 ms)3 split writer threads finished at 1731407412285Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testSequentialEditLogSeqNum/1d7adfde8f7246cc0322536ae9f0d55a/recovered.edits/0000000000000000003-wal.1731407412157.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731407412294 (+9 ms)Rename recovered edits hdfs://localhost:38657/hbase/data/default/testSequentialEditLogSeqNum/1d7adfde8f7246cc0322536ae9f0d55a/recovered.edits/0000000000000000003-wal.1731407412157.temp to hdfs://localhost:38657/hbase/data/default/testSequentialEditLogSeqNum/1d7adfde8f7246cc0322536ae9f0d55a/recovered.edits/0000000000000000020 at 1731407412295 (+1 ms)Processed 17 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:38657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731407412150/wal.1731407412157, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1731407412295 2024-11-12T10:30:12,311 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=407 (was 402) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:35818 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:46624 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=781 (was 781), ProcessCount=11 (was 11), AvailableMemoryMB=7482 (was 7501) 2024-11-12T10:30:12,324 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=407, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=781, ProcessCount=11, AvailableMemoryMB=7481 2024-11-12T10:30:12,343 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:12,345 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:12,377 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T10:30:12,380 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-46143038, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/hregion-46143038, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:12,392 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-46143038/hregion-46143038.1731407412380, exclude list is [], retry=0 2024-11-12T10:30:12,395 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:12,395 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:12,395 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:12,399 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-46143038/hregion-46143038.1731407412380 2024-11-12T10:30:12,399 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:39823:39823)] 2024-11-12T10:30:12,399 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 59fcca45db43e970ae79fd0f8a15992e, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731407412343.59fcca45db43e970ae79fd0f8a15992e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38657/hbase 2024-11-12T10:30:12,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741879_1056 (size=70) 2024-11-12T10:30:12,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741879_1056 (size=70) 2024-11-12T10:30:12,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741879_1056 (size=70) 2024-11-12T10:30:12,409 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731407412343.59fcca45db43e970ae79fd0f8a15992e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:12,412 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,414 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59fcca45db43e970ae79fd0f8a15992e columnFamilyName a 2024-11-12T10:30:12,414 DEBUG [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:12,415 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] regionserver.HStore(327): Store=59fcca45db43e970ae79fd0f8a15992e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:12,415 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,417 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59fcca45db43e970ae79fd0f8a15992e columnFamilyName b 2024-11-12T10:30:12,417 DEBUG [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:12,417 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] regionserver.HStore(327): Store=59fcca45db43e970ae79fd0f8a15992e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:12,417 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,419 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59fcca45db43e970ae79fd0f8a15992e columnFamilyName c 2024-11-12T10:30:12,419 DEBUG [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:12,419 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] regionserver.HStore(327): Store=59fcca45db43e970ae79fd0f8a15992e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:12,419 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,420 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,421 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,422 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,423 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,423 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T10:30:12,425 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,428 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:30:12,428 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 59fcca45db43e970ae79fd0f8a15992e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66892028, jitterRate=-0.0032311081886291504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T10:30:12,430 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 59fcca45db43e970ae79fd0f8a15992e: Writing region info on filesystem at 1731407412409Initializing all the Stores at 1731407412410 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407412410Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407412412 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407412412Cleaning up temporary data from old regions at 1731407412423 (+11 ms)Region opened successfully at 1731407412429 (+6 ms) 2024-11-12T10:30:12,430 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 59fcca45db43e970ae79fd0f8a15992e, disabling compactions & flushes 2024-11-12T10:30:12,430 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731407412343.59fcca45db43e970ae79fd0f8a15992e. 2024-11-12T10:30:12,430 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731407412343.59fcca45db43e970ae79fd0f8a15992e. 2024-11-12T10:30:12,430 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731407412343.59fcca45db43e970ae79fd0f8a15992e. after waiting 0 ms 2024-11-12T10:30:12,430 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731407412343.59fcca45db43e970ae79fd0f8a15992e. 2024-11-12T10:30:12,430 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731407412343.59fcca45db43e970ae79fd0f8a15992e. 2024-11-12T10:30:12,430 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 59fcca45db43e970ae79fd0f8a15992e: Waiting for close lock at 1731407412430Disabling compacts and flushes for region at 1731407412430Disabling writes for close at 1731407412430Writing region close event to WAL at 1731407412430Closed at 1731407412430 2024-11-12T10:30:12,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741878_1055 (size=93) 2024-11-12T10:30:12,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741878_1055 (size=93) 2024-11-12T10:30:12,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741878_1055 (size=93) 2024-11-12T10:30:12,436 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T10:30:12,436 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-46143038:(num 1731407412380) 2024-11-12T10:30:12,436 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:12,439 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:12,458 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407412439, exclude list is [], retry=0 2024-11-12T10:30:12,462 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:12,462 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:12,463 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:12,464 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407412439 2024-11-12T10:30:12,465 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33367:33367)] 2024-11-12T10:30:12,465 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 59fcca45db43e970ae79fd0f8a15992e, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731407412343.59fcca45db43e970ae79fd0f8a15992e.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:12,465 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731407412343.59fcca45db43e970ae79fd0f8a15992e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:12,466 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,466 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,467 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,469 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59fcca45db43e970ae79fd0f8a15992e columnFamilyName a 2024-11-12T10:30:12,469 DEBUG [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:12,470 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] regionserver.HStore(327): Store=59fcca45db43e970ae79fd0f8a15992e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:12,470 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,471 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59fcca45db43e970ae79fd0f8a15992e columnFamilyName b 2024-11-12T10:30:12,471 DEBUG [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:12,472 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] regionserver.HStore(327): Store=59fcca45db43e970ae79fd0f8a15992e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:12,472 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,473 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59fcca45db43e970ae79fd0f8a15992e columnFamilyName c 2024-11-12T10:30:12,473 DEBUG [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:12,474 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] regionserver.HStore(327): Store=59fcca45db43e970ae79fd0f8a15992e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:12,474 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,475 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,476 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,478 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,478 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,478 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T10:30:12,480 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:12,482 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 59fcca45db43e970ae79fd0f8a15992e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68691908, jitterRate=0.02358919382095337}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T10:30:12,483 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 59fcca45db43e970ae79fd0f8a15992e: Writing region info on filesystem at 1731407412466Initializing all the Stores at 1731407412467 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407412467Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407412467Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407412467Cleaning up temporary data from old regions at 1731407412478 (+11 ms)Region opened successfully at 1731407412483 (+5 ms) 2024-11-12T10:30:12,488 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731407412487/Put/seqid=0 2024-11-12T10:30:12,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741881_1058 (size=4826) 2024-11-12T10:30:12,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741881_1058 (size=4826) 2024-11-12T10:30:12,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741881_1058 (size=4826) 2024-11-12T10:30:12,498 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38657/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 59fcca45db43e970ae79fd0f8a15992e/a 2024-11-12T10:30:12,507 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-12T10:30:12,507 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-12T10:30:12,507 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 59fcca45db43e970ae79fd0f8a15992e: 2024-11-12T10:30:12,509 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/a/6e191e9d337d423a87f0788c897e6c37_SeqId_3_ 2024-11-12T10:30:12,510 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38657/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 59fcca45db43e970ae79fd0f8a15992e/a as hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/a/6e191e9d337d423a87f0788c897e6c37_SeqId_3_ - updating store file list. 2024-11-12T10:30:12,516 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6e191e9d337d423a87f0788c897e6c37_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-12T10:30:12,517 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/a/6e191e9d337d423a87f0788c897e6c37_SeqId_3_ into 59fcca45db43e970ae79fd0f8a15992e/a 2024-11-12T10:30:12,517 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38657/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 59fcca45db43e970ae79fd0f8a15992e/a (new location: hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/a/6e191e9d337d423a87f0788c897e6c37_SeqId_3_) 2024-11-12T10:30:12,558 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407412439, size=0 (0bytes) 2024-11-12T10:30:12,559 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407412439 might be still open, length is 0 2024-11-12T10:30:12,559 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407412439 2024-11-12T10:30:12,559 WARN [IPC Server handler 2 on default port 38657 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407412439 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-12T10:30:12,559 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407412439 after 0ms 2024-11-12T10:30:12,720 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:49776 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:35741:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49776 dst: /127.0.0.1:35741 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35741 remote=/127.0.0.1:49776]. Total timeout mills is 60000, 59802 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:12,722 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:46718 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:34283:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46718 dst: /127.0.0.1:34283 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:12,722 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:47030 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43905:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47030 dst: /127.0.0.1:43905 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:12,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741880_1059 (size=504) 2024-11-12T10:30:12,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741880_1059 (size=504) 2024-11-12T10:30:16,561 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407412439 after 4002ms 2024-11-12T10:30:16,571 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407412439 took 4012ms 2024-11-12T10:30:16,572 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407412439; continuing. 2024-11-12T10:30:16,572 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407412439 so closing down 2024-11-12T10:30:16,572 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T10:30:16,574 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731407412439.temp 2024-11-12T10:30:16,575 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/recovered.edits/0000000000000000005-wal.1731407412439.temp 2024-11-12T10:30:16,575 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T10:30:16,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741882_1060 (size=238) 2024-11-12T10:30:16,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741882_1060 (size=238) 2024-11-12T10:30:16,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741882_1060 (size=238) 2024-11-12T10:30:16,582 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/recovered.edits/0000000000000000005-wal.1731407412439.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-12T10:30:16,584 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/recovered.edits/0000000000000000005-wal.1731407412439.temp to hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/recovered.edits/0000000000000000005 2024-11-12T10:30:16,584 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 13 ms; skipped=1; WAL=hdfs://localhost:38657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407412439, size=0, length=0, corrupted=false, cancelled=false 2024-11-12T10:30:16,584 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407412439, journal: Splitting hdfs://localhost:38657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407412439, size=0 (0bytes) at 1731407412558Finishing writing output for hdfs://localhost:38657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407412439 so closing down at 1731407416572 (+4014 ms)Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/recovered.edits/0000000000000000005-wal.1731407412439.temp at 1731407416575 (+3 ms)3 split writer threads finished at 1731407416575Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/recovered.edits/0000000000000000005-wal.1731407412439.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731407416582 (+7 ms)Rename recovered edits hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/recovered.edits/0000000000000000005-wal.1731407412439.temp to hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/recovered.edits/0000000000000000005 at 1731407416584 (+2 ms)Processed 2 edits across 1 Regions in 13 ms; skipped=1; WAL=hdfs://localhost:38657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407412439, size=0, length=0, corrupted=false, cancelled=false at 1731407416584 2024-11-12T10:30:16,586 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407412439 to hdfs://localhost:38657/hbase/oldWALs/wal.1731407412439 2024-11-12T10:30:16,587 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/recovered.edits/0000000000000000005 2024-11-12T10:30:16,587 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:16,589 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:16,602 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407416589, exclude list is [], retry=0 2024-11-12T10:30:16,605 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:16,605 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:16,605 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:16,607 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407416589 2024-11-12T10:30:16,608 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:39823:39823)] 2024-11-12T10:30:16,608 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 59fcca45db43e970ae79fd0f8a15992e, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731407412343.59fcca45db43e970ae79fd0f8a15992e.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:16,608 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731407412343.59fcca45db43e970ae79fd0f8a15992e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:16,608 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:16,608 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:16,610 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:16,611 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59fcca45db43e970ae79fd0f8a15992e columnFamilyName a 2024-11-12T10:30:16,611 DEBUG [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:16,618 DEBUG [StoreFileOpener-59fcca45db43e970ae79fd0f8a15992e-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6e191e9d337d423a87f0788c897e6c37_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-12T10:30:16,618 DEBUG [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/a/6e191e9d337d423a87f0788c897e6c37_SeqId_3_ 2024-11-12T10:30:16,618 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] regionserver.HStore(327): Store=59fcca45db43e970ae79fd0f8a15992e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:16,618 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:16,620 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59fcca45db43e970ae79fd0f8a15992e columnFamilyName b 2024-11-12T10:30:16,620 DEBUG [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:16,620 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] regionserver.HStore(327): Store=59fcca45db43e970ae79fd0f8a15992e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:16,620 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:16,621 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59fcca45db43e970ae79fd0f8a15992e columnFamilyName c 2024-11-12T10:30:16,621 DEBUG [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:16,622 INFO [StoreOpener-59fcca45db43e970ae79fd0f8a15992e-1 {}] regionserver.HStore(327): Store=59fcca45db43e970ae79fd0f8a15992e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:16,622 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:16,623 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:16,625 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:16,626 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/recovered.edits/0000000000000000005 2024-11-12T10:30:16,629 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/recovered.edits/0000000000000000005 2024-11-12T10:30:16,629 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 59fcca45db43e970ae79fd0f8a15992e 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-12T10:30:16,643 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/.tmp/a/6de560f4bb8c4c6ebf6e7c53124bcdf3 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731407412521/Put/seqid=0 2024-11-12T10:30:16,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741884_1062 (size=5149) 2024-11-12T10:30:16,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741884_1062 (size=5149) 2024-11-12T10:30:16,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741884_1062 (size=5149) 2024-11-12T10:30:16,649 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/.tmp/a/6de560f4bb8c4c6ebf6e7c53124bcdf3 2024-11-12T10:30:16,655 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/.tmp/a/6de560f4bb8c4c6ebf6e7c53124bcdf3 as hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/a/6de560f4bb8c4c6ebf6e7c53124bcdf3 2024-11-12T10:30:16,661 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/a/6de560f4bb8c4c6ebf6e7c53124bcdf3, entries=1, sequenceid=5, filesize=5.0 K 2024-11-12T10:30:16,661 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 59fcca45db43e970ae79fd0f8a15992e in 32ms, sequenceid=5, compaction requested=false; wal=null 2024-11-12T10:30:16,662 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/recovered.edits/0000000000000000005 2024-11-12T10:30:16,663 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:16,663 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:16,664 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T10:30:16,666 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 59fcca45db43e970ae79fd0f8a15992e 2024-11-12T10:30:16,669 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/59fcca45db43e970ae79fd0f8a15992e/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-12T10:30:16,670 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 59fcca45db43e970ae79fd0f8a15992e; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73472948, jitterRate=0.09483224153518677}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T10:30:16,671 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 59fcca45db43e970ae79fd0f8a15992e: Writing region info on filesystem at 1731407416609Initializing all the Stores at 1731407416610 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407416610Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407416610Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407416610Obtaining lock to block concurrent updates at 1731407416629 (+19 ms)Preparing flush snapshotting stores in 59fcca45db43e970ae79fd0f8a15992e at 1731407416629Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731407412343.59fcca45db43e970ae79fd0f8a15992e., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731407416629Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731407412343.59fcca45db43e970ae79fd0f8a15992e. at 1731407416629Flushing 59fcca45db43e970ae79fd0f8a15992e/a: creating writer at 1731407416629Flushing 59fcca45db43e970ae79fd0f8a15992e/a: appending metadata at 1731407416642 (+13 ms)Flushing 59fcca45db43e970ae79fd0f8a15992e/a: closing flushed file at 1731407416642Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21138ba4: reopening flushed file at 1731407416654 (+12 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 59fcca45db43e970ae79fd0f8a15992e in 32ms, sequenceid=5, compaction requested=false; wal=null at 1731407416661 (+7 ms)Cleaning up temporary data from old regions at 1731407416664 (+3 ms)Region opened successfully at 1731407416670 (+6 ms) 2024-11-12T10:30:16,675 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 59fcca45db43e970ae79fd0f8a15992e, disabling compactions & flushes 2024-11-12T10:30:16,675 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731407412343.59fcca45db43e970ae79fd0f8a15992e. 2024-11-12T10:30:16,675 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731407412343.59fcca45db43e970ae79fd0f8a15992e. 2024-11-12T10:30:16,675 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731407412343.59fcca45db43e970ae79fd0f8a15992e. after waiting 0 ms 2024-11-12T10:30:16,675 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731407412343.59fcca45db43e970ae79fd0f8a15992e. 2024-11-12T10:30:16,676 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731407412343.59fcca45db43e970ae79fd0f8a15992e. 2024-11-12T10:30:16,676 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 59fcca45db43e970ae79fd0f8a15992e: Waiting for close lock at 1731407416675Disabling compacts and flushes for region at 1731407416675Disabling writes for close at 1731407416675Writing region close event to WAL at 1731407416676 (+1 ms)Closed at 1731407416676 2024-11-12T10:30:16,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741883_1061 (size=93) 2024-11-12T10:30:16,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741883_1061 (size=93) 2024-11-12T10:30:16,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741883_1061 (size=93) 2024-11-12T10:30:16,682 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T10:30:16,682 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731407416589) 2024-11-12T10:30:16,698 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=411 (was 407) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:38657 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-47896843_22 at /127.0.0.1:49818 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-47896843_22 at /127.0.0.1:47046 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (865318088) connection to localhost/127.0.0.1:38657 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1022 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=735 (was 781), ProcessCount=11 (was 11), AvailableMemoryMB=7427 (was 7481) 2024-11-12T10:30:16,712 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=411, OpenFileDescriptor=1022, MaxFileDescriptor=1048576, SystemLoadAverage=735, ProcessCount=11, AvailableMemoryMB=7424 2024-11-12T10:30:16,727 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:16,733 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T10:30:16,737 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a10fb4296bab,35883,1731407401663 2024-11-12T10:30:16,739 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7b1be2da 2024-11-12T10:30:16,740 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T10:30:16,742 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T10:30:16,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35883 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T10:30:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35883 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-12T10:30:16,753 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T10:30:16,755 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35883 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-12T10:30:16,755 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:16,757 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T10:30:16,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T10:30:16,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741885_1063 (size=694) 2024-11-12T10:30:16,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741885_1063 (size=694) 2024-11-12T10:30:16,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741885_1063 (size=694) 2024-11-12T10:30:16,773 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eaa006ae0460d9d2606f62df60fe50d7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf 2024-11-12T10:30:16,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741886_1064 (size=77) 2024-11-12T10:30:16,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741886_1064 (size=77) 2024-11-12T10:30:16,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741886_1064 (size=77) 2024-11-12T10:30:16,784 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:16,784 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing eaa006ae0460d9d2606f62df60fe50d7, disabling compactions & flushes 2024-11-12T10:30:16,784 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:16,784 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:16,785 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. after waiting 0 ms 2024-11-12T10:30:16,785 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:16,785 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:16,785 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for eaa006ae0460d9d2606f62df60fe50d7: Waiting for close lock at 1731407416784Disabling compacts and flushes for region at 1731407416784Disabling writes for close at 1731407416785 (+1 ms)Writing region close event to WAL at 1731407416785Closed at 1731407416785 2024-11-12T10:30:16,786 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T10:30:16,794 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731407416787"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731407416787"}]},"ts":"1731407416787"} 2024-11-12T10:30:16,798 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T10:30:16,800 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T10:30:16,803 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731407416800"}]},"ts":"1731407416800"} 2024-11-12T10:30:16,807 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-12T10:30:16,808 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {a10fb4296bab=0} racks are {/default-rack=0} 2024-11-12T10:30:16,809 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T10:30:16,809 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T10:30:16,809 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-12T10:30:16,809 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T10:30:16,809 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T10:30:16,809 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-12T10:30:16,809 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T10:30:16,809 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T10:30:16,809 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-12T10:30:16,809 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T10:30:16,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, ASSIGN}] 2024-11-12T10:30:16,813 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, ASSIGN 2024-11-12T10:30:16,814 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, ASSIGN; state=OFFLINE, location=a10fb4296bab,37011,1731407402537; forceNewPlan=false, retain=false 2024-11-12T10:30:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T10:30:16,966 INFO [a10fb4296bab:35883 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-12T10:30:16,967 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eaa006ae0460d9d2606f62df60fe50d7, regionState=OPENING, regionLocation=a10fb4296bab,37011,1731407402537 2024-11-12T10:30:16,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, ASSIGN because future has completed 2024-11-12T10:30:16,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaa006ae0460d9d2606f62df60fe50d7, server=a10fb4296bab,37011,1731407402537}] 2024-11-12T10:30:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T10:30:17,132 INFO [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:17,132 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => eaa006ae0460d9d2606f62df60fe50d7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:17,133 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,133 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:17,133 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,133 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,136 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,138 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaa006ae0460d9d2606f62df60fe50d7 columnFamilyName cf1 2024-11-12T10:30:17,138 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:17,139 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(327): Store=eaa006ae0460d9d2606f62df60fe50d7/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:17,139 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,142 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaa006ae0460d9d2606f62df60fe50d7 columnFamilyName cf2 2024-11-12T10:30:17,142 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:17,142 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(327): Store=eaa006ae0460d9d2606f62df60fe50d7/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:17,143 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,144 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,144 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,145 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,145 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,146 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-12T10:30:17,148 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,153 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:30:17,154 INFO [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened eaa006ae0460d9d2606f62df60fe50d7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65436635, jitterRate=-0.024918153882026672}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-12T10:30:17,154 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,155 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for eaa006ae0460d9d2606f62df60fe50d7: Running coprocessor pre-open hook at 1731407417134Writing region info on filesystem at 1731407417134Initializing all the Stores at 1731407417135 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407417135Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407417135Cleaning up temporary data from old regions at 1731407417145 (+10 ms)Running coprocessor post-open hooks at 1731407417154 (+9 ms)Region opened successfully at 1731407417155 (+1 ms) 2024-11-12T10:30:17,157 INFO [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., pid=6, masterSystemTime=1731407417125 2024-11-12T10:30:17,161 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:17,161 INFO [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:17,162 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eaa006ae0460d9d2606f62df60fe50d7, regionState=OPEN, openSeqNum=2, regionLocation=a10fb4296bab,37011,1731407402537 2024-11-12T10:30:17,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaa006ae0460d9d2606f62df60fe50d7, server=a10fb4296bab,37011,1731407402537 because future has completed 2024-11-12T10:30:17,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-12T10:30:17,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure eaa006ae0460d9d2606f62df60fe50d7, server=a10fb4296bab,37011,1731407402537 in 196 msec 2024-11-12T10:30:17,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-12T10:30:17,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, ASSIGN in 360 msec 2024-11-12T10:30:17,177 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T10:30:17,177 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731407417177"}]},"ts":"1731407417177"} 2024-11-12T10:30:17,180 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-12T10:30:17,182 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T10:30:17,185 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 435 msec 2024-11-12T10:30:17,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T10:30:17,393 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-12T10:30:17,393 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-12T10:30:17,395 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T10:30:17,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-12T10:30:17,403 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T10:30:17,404 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-12T10:30:17,416 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=2] 2024-11-12T10:30:17,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35883 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=eaa006ae0460d9d2606f62df60fe50d7, source=a10fb4296bab,37011,1731407402537, destination=a10fb4296bab,45327,1731407402438, warming up region on a10fb4296bab,45327,1731407402438 2024-11-12T10:30:17,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35883 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T10:30:17,438 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35883 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=eaa006ae0460d9d2606f62df60fe50d7, source=a10fb4296bab,37011,1731407402537, destination=a10fb4296bab,45327,1731407402438, running balancer 2024-11-12T10:30:17,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36695, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T10:30:17,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35883 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, REOPEN/MOVE 2024-11-12T10:30:17,440 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, REOPEN/MOVE 2024-11-12T10:30:17,443 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35883 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=a10fb4296bab,37011,1731407402537, table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T10:30:17,443 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=eaa006ae0460d9d2606f62df60fe50d7, regionState=CLOSING, regionLocation=a10fb4296bab,37011,1731407402537 2024-11-12T10:30:17,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45327 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:17,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45327 {}] regionserver.HRegion(7855): Warmup {ENCODED => eaa006ae0460d9d2606f62df60fe50d7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:17,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45327 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:17,446 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, REOPEN/MOVE because future has completed 2024-11-12T10:30:17,447 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T10:30:17,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure eaa006ae0460d9d2606f62df60fe50d7, server=a10fb4296bab,37011,1731407402537}] 2024-11-12T10:30:17,447 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaa006ae0460d9d2606f62df60fe50d7 columnFamilyName cf1 2024-11-12T10:30:17,447 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:17,448 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(327): Store=eaa006ae0460d9d2606f62df60fe50d7/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:17,448 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,449 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaa006ae0460d9d2606f62df60fe50d7 columnFamilyName cf2 2024-11-12T10:30:17,449 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:17,450 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(327): Store=eaa006ae0460d9d2606f62df60fe50d7/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:17,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45327 {}] regionserver.HRegion(1722): Closing eaa006ae0460d9d2606f62df60fe50d7, disabling compactions & flushes 2024-11-12T10:30:17,450 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45327 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:17,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45327 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:17,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45327 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. after waiting 0 ms 2024-11-12T10:30:17,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45327 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:17,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45327 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:17,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45327 {}] regionserver.HRegion(1676): Region close journal for eaa006ae0460d9d2606f62df60fe50d7: Waiting for close lock at 1731407417450Disabling compacts and flushes for region at 1731407417450Disabling writes for close at 1731407417450Writing region close event to WAL at 1731407417451 (+1 ms)Closed at 1731407417451 2024-11-12T10:30:17,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35883 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-12T10:30:17,608 INFO [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,608 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T10:30:17,609 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing eaa006ae0460d9d2606f62df60fe50d7, disabling compactions & flushes 2024-11-12T10:30:17,610 INFO [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:17,610 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:17,610 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. after waiting 0 ms 2024-11-12T10:30:17,610 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:17,610 INFO [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing eaa006ae0460d9d2606f62df60fe50d7 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-12T10:30:17,631 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/.tmp/cf1/f523ccd65051495cb7b4254bc8d69ac5 is 35, key is r1/cf1:q/1731407417419/Put/seqid=0 2024-11-12T10:30:17,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741887_1065 (size=4783) 2024-11-12T10:30:17,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741887_1065 (size=4783) 2024-11-12T10:30:17,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741887_1065 (size=4783) 2024-11-12T10:30:17,637 INFO [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/.tmp/cf1/f523ccd65051495cb7b4254bc8d69ac5 2024-11-12T10:30:17,644 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/.tmp/cf1/f523ccd65051495cb7b4254bc8d69ac5 as hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf1/f523ccd65051495cb7b4254bc8d69ac5 2024-11-12T10:30:17,651 INFO [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf1/f523ccd65051495cb7b4254bc8d69ac5, entries=1, sequenceid=5, filesize=4.7 K 2024-11-12T10:30:17,652 INFO [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for eaa006ae0460d9d2606f62df60fe50d7 in 42ms, sequenceid=5, compaction requested=false 2024-11-12T10:30:17,652 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-12T10:30:17,657 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T10:30:17,659 INFO [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:17,659 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for eaa006ae0460d9d2606f62df60fe50d7: Waiting for close lock at 1731407417609Running coprocessor pre-close hooks at 1731407417609Disabling compacts and flushes for region at 1731407417609Disabling writes for close at 1731407417610 (+1 ms)Obtaining lock to block concurrent updates at 1731407417610Preparing flush snapshotting stores in eaa006ae0460d9d2606f62df60fe50d7 at 1731407417610Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731407417611 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. at 1731407417613 (+2 ms)Flushing eaa006ae0460d9d2606f62df60fe50d7/cf1: creating writer at 1731407417613Flushing eaa006ae0460d9d2606f62df60fe50d7/cf1: appending metadata at 1731407417631 (+18 ms)Flushing eaa006ae0460d9d2606f62df60fe50d7/cf1: closing flushed file at 1731407417631Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dfb99f1: reopening flushed file at 1731407417643 (+12 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for eaa006ae0460d9d2606f62df60fe50d7 in 42ms, sequenceid=5, compaction requested=false at 1731407417652 (+9 ms)Writing region close event to WAL at 1731407417654 (+2 ms)Running coprocessor post-close hooks at 1731407417657 (+3 ms)Closed at 1731407417659 (+2 ms) 2024-11-12T10:30:17,660 INFO [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding eaa006ae0460d9d2606f62df60fe50d7 move to a10fb4296bab,45327,1731407402438 record at close sequenceid=5 2024-11-12T10:30:17,663 INFO [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,663 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=eaa006ae0460d9d2606f62df60fe50d7, regionState=CLOSED 2024-11-12T10:30:17,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure eaa006ae0460d9d2606f62df60fe50d7, server=a10fb4296bab,37011,1731407402537 because future has completed 2024-11-12T10:30:17,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-12T10:30:17,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure eaa006ae0460d9d2606f62df60fe50d7, server=a10fb4296bab,37011,1731407402537 in 220 msec 2024-11-12T10:30:17,670 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, REOPEN/MOVE; state=CLOSED, location=a10fb4296bab,45327,1731407402438; forceNewPlan=false, retain=false 2024-11-12T10:30:17,821 INFO [a10fb4296bab:35883 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-12T10:30:17,821 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=eaa006ae0460d9d2606f62df60fe50d7, regionState=OPENING, regionLocation=a10fb4296bab,45327,1731407402438 2024-11-12T10:30:17,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, REOPEN/MOVE because future has completed 2024-11-12T10:30:17,825 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaa006ae0460d9d2606f62df60fe50d7, server=a10fb4296bab,45327,1731407402438}] 2024-11-12T10:30:17,986 INFO [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:17,986 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => eaa006ae0460d9d2606f62df60fe50d7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:17,987 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,987 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:17,987 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,988 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,991 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:17,994 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaa006ae0460d9d2606f62df60fe50d7 columnFamilyName cf1 2024-11-12T10:30:17,994 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:18,002 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf1/f523ccd65051495cb7b4254bc8d69ac5 2024-11-12T10:30:18,002 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(327): Store=eaa006ae0460d9d2606f62df60fe50d7/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:18,002 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,003 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaa006ae0460d9d2606f62df60fe50d7 columnFamilyName cf2 2024-11-12T10:30:18,003 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:18,004 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(327): Store=eaa006ae0460d9d2606f62df60fe50d7/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:18,004 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,005 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,007 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,008 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,008 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,009 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-12T10:30:18,011 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,012 INFO [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened eaa006ae0460d9d2606f62df60fe50d7; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73523833, jitterRate=0.0955904871225357}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-12T10:30:18,013 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,014 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for eaa006ae0460d9d2606f62df60fe50d7: Running coprocessor pre-open hook at 1731407417988Writing region info on filesystem at 1731407417988Initializing all the Stores at 1731407417990 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407417991 (+1 ms)Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407417991Cleaning up temporary data from old regions at 1731407418008 (+17 ms)Running coprocessor post-open hooks at 1731407418013 (+5 ms)Region opened successfully at 1731407418014 (+1 ms) 2024-11-12T10:30:18,036 INFO [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., pid=9, masterSystemTime=1731407417978 2024-11-12T10:30:18,038 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:18,039 INFO [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:18,039 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=eaa006ae0460d9d2606f62df60fe50d7, regionState=OPEN, openSeqNum=9, regionLocation=a10fb4296bab,45327,1731407402438 2024-11-12T10:30:18,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaa006ae0460d9d2606f62df60fe50d7, server=a10fb4296bab,45327,1731407402438 because future has completed 2024-11-12T10:30:18,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-12T10:30:18,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure eaa006ae0460d9d2606f62df60fe50d7, server=a10fb4296bab,45327,1731407402438 in 219 msec 2024-11-12T10:30:18,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, REOPEN/MOVE in 608 msec 2024-11-12T10:30:18,096 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T10:30:18,098 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T10:30:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37011 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:38868 deadline: 1731407478102, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a10fb4296bab port=45327 startCode=1731407402438. As of locationSeqNum=5. 2024-11-12T10:30:18,123 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a10fb4296bab port=45327 startCode=1731407402438. As of locationSeqNum=5. 2024-11-12T10:30:18,124 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a10fb4296bab port=45327 startCode=1731407402438. As of locationSeqNum=5. 2024-11-12T10:30:18,124 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,45327,1731407402438, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a10fb4296bab port=45327 startCode=1731407402438. As of locationSeqNum=5. 2024-11-12T10:30:18,233 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:30:18,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:30:18,248 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing eaa006ae0460d9d2606f62df60fe50d7 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-12T10:30:18,263 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/.tmp/cf1/3fdce4a56f2f487fb98fd3e11ccaef69 is 29, key is r1/cf1:/1731407418238/DeleteFamily/seqid=0 2024-11-12T10:30:18,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741888_1066 (size=4906) 2024-11-12T10:30:18,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741888_1066 (size=4906) 2024-11-12T10:30:18,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741888_1066 (size=4906) 2024-11-12T10:30:18,270 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/.tmp/cf1/3fdce4a56f2f487fb98fd3e11ccaef69 2024-11-12T10:30:18,277 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3fdce4a56f2f487fb98fd3e11ccaef69 2024-11-12T10:30:18,297 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/.tmp/cf2/d00170be72a44d51b8524ceeb3cd2b84 is 29, key is r1/cf2:/1731407418238/DeleteFamily/seqid=0 2024-11-12T10:30:18,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741889_1067 (size=4906) 2024-11-12T10:30:18,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741889_1067 (size=4906) 2024-11-12T10:30:18,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741889_1067 (size=4906) 2024-11-12T10:30:18,304 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/.tmp/cf2/d00170be72a44d51b8524ceeb3cd2b84 2024-11-12T10:30:18,310 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d00170be72a44d51b8524ceeb3cd2b84 2024-11-12T10:30:18,311 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/.tmp/cf1/3fdce4a56f2f487fb98fd3e11ccaef69 as hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf1/3fdce4a56f2f487fb98fd3e11ccaef69 2024-11-12T10:30:18,319 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3fdce4a56f2f487fb98fd3e11ccaef69 2024-11-12T10:30:18,319 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf1/3fdce4a56f2f487fb98fd3e11ccaef69, entries=1, sequenceid=12, filesize=4.8 K 2024-11-12T10:30:18,320 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/.tmp/cf2/d00170be72a44d51b8524ceeb3cd2b84 as hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf2/d00170be72a44d51b8524ceeb3cd2b84 2024-11-12T10:30:18,326 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d00170be72a44d51b8524ceeb3cd2b84 2024-11-12T10:30:18,327 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf2/d00170be72a44d51b8524ceeb3cd2b84, entries=1, sequenceid=12, filesize=4.8 K 2024-11-12T10:30:18,328 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for eaa006ae0460d9d2606f62df60fe50d7 in 80ms, sequenceid=12, compaction requested=false 2024-11-12T10:30:18,328 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for eaa006ae0460d9d2606f62df60fe50d7: 2024-11-12T10:30:18,330 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-12T10:30:18,331 DEBUG [Time-limited test {}] regionserver.HStore(1541): eaa006ae0460d9d2606f62df60fe50d7/cf1 is initiating major compaction (all files) 2024-11-12T10:30:18,331 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T10:30:18,331 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:18,331 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of eaa006ae0460d9d2606f62df60fe50d7/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:18,332 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf1/f523ccd65051495cb7b4254bc8d69ac5, hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf1/3fdce4a56f2f487fb98fd3e11ccaef69] into tmpdir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/.tmp, totalSize=9.5 K 2024-11-12T10:30:18,333 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f523ccd65051495cb7b4254bc8d69ac5, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731407417419 2024-11-12T10:30:18,333 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3fdce4a56f2f487fb98fd3e11ccaef69, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-12T10:30:18,345 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): eaa006ae0460d9d2606f62df60fe50d7#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:30:18,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741890_1068 (size=4626) 2024-11-12T10:30:18,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741890_1068 (size=4626) 2024-11-12T10:30:18,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741890_1068 (size=4626) 2024-11-12T10:30:18,361 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/.tmp/cf1/51d72d268d604d5d8b19f68a4a5b82f8 as hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf1/51d72d268d604d5d8b19f68a4a5b82f8 2024-11-12T10:30:18,375 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in eaa006ae0460d9d2606f62df60fe50d7/cf1 of eaa006ae0460d9d2606f62df60fe50d7 into 51d72d268d604d5d8b19f68a4a5b82f8(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:30:18,375 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for eaa006ae0460d9d2606f62df60fe50d7: 2024-11-12T10:30:18,375 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-12T10:30:18,375 DEBUG [Time-limited test {}] regionserver.HStore(1541): eaa006ae0460d9d2606f62df60fe50d7/cf2 is initiating major compaction (all files) 2024-11-12T10:30:18,375 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T10:30:18,375 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T10:30:18,375 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of eaa006ae0460d9d2606f62df60fe50d7/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:18,375 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf2/d00170be72a44d51b8524ceeb3cd2b84] into tmpdir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/.tmp, totalSize=4.8 K 2024-11-12T10:30:18,376 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d00170be72a44d51b8524ceeb3cd2b84, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-12T10:30:18,382 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): eaa006ae0460d9d2606f62df60fe50d7#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T10:30:18,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741891_1069 (size=4592) 2024-11-12T10:30:18,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741891_1069 (size=4592) 2024-11-12T10:30:18,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741891_1069 (size=4592) 2024-11-12T10:30:18,397 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/.tmp/cf2/4f3a8aef788d431399dd31218532ab7c as hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf2/4f3a8aef788d431399dd31218532ab7c 2024-11-12T10:30:18,404 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in eaa006ae0460d9d2606f62df60fe50d7/cf2 of eaa006ae0460d9d2606f62df60fe50d7 into 4f3a8aef788d431399dd31218532ab7c(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:30:18,404 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for eaa006ae0460d9d2606f62df60fe50d7: 2024-11-12T10:30:18,413 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35883 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=eaa006ae0460d9d2606f62df60fe50d7, source=a10fb4296bab,45327,1731407402438, destination=a10fb4296bab,37011,1731407402537, warming up region on a10fb4296bab,37011,1731407402537 2024-11-12T10:30:18,413 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35883 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=eaa006ae0460d9d2606f62df60fe50d7, source=a10fb4296bab,45327,1731407402438, destination=a10fb4296bab,37011,1731407402537, running balancer 2024-11-12T10:30:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35883 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, REOPEN/MOVE 2024-11-12T10:30:18,415 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, REOPEN/MOVE 2024-11-12T10:30:18,416 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=eaa006ae0460d9d2606f62df60fe50d7, regionState=CLOSING, regionLocation=a10fb4296bab,45327,1731407402438 2024-11-12T10:30:18,417 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37011 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:18,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37011 {}] regionserver.HRegion(7855): Warmup {ENCODED => eaa006ae0460d9d2606f62df60fe50d7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:18,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37011 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:18,418 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, REOPEN/MOVE because future has completed 2024-11-12T10:30:18,419 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaa006ae0460d9d2606f62df60fe50d7 columnFamilyName cf1 2024-11-12T10:30:18,420 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:18,420 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T10:30:18,420 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure eaa006ae0460d9d2606f62df60fe50d7, server=a10fb4296bab,45327,1731407402438}] 2024-11-12T10:30:18,428 INFO [StoreFileOpener-eaa006ae0460d9d2606f62df60fe50d7-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3fdce4a56f2f487fb98fd3e11ccaef69 2024-11-12T10:30:18,428 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf1/3fdce4a56f2f487fb98fd3e11ccaef69 2024-11-12T10:30:18,434 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf1/51d72d268d604d5d8b19f68a4a5b82f8 2024-11-12T10:30:18,439 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf1/f523ccd65051495cb7b4254bc8d69ac5 2024-11-12T10:30:18,440 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(327): Store=eaa006ae0460d9d2606f62df60fe50d7/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:18,440 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,441 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaa006ae0460d9d2606f62df60fe50d7 columnFamilyName cf2 2024-11-12T10:30:18,441 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:18,449 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf2/4f3a8aef788d431399dd31218532ab7c 2024-11-12T10:30:18,455 INFO [StoreFileOpener-eaa006ae0460d9d2606f62df60fe50d7-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d00170be72a44d51b8524ceeb3cd2b84 2024-11-12T10:30:18,455 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf2/d00170be72a44d51b8524ceeb3cd2b84 2024-11-12T10:30:18,455 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(327): Store=eaa006ae0460d9d2606f62df60fe50d7/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37011 {}] regionserver.HRegion(1722): Closing eaa006ae0460d9d2606f62df60fe50d7, disabling compactions & flushes 2024-11-12T10:30:18,455 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37011 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37011 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37011 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. after waiting 0 ms 2024-11-12T10:30:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37011 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:18,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37011 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37011 {}] regionserver.HRegion(1676): Region close journal for eaa006ae0460d9d2606f62df60fe50d7: Waiting for close lock at 1731407418455Disabling compacts and flushes for region at 1731407418455Disabling writes for close at 1731407418456 (+1 ms)Writing region close event to WAL at 1731407418457 (+1 ms)Closed at 1731407418457 2024-11-12T10:30:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35883 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-12T10:30:18,575 INFO [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,575 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T10:30:18,575 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing eaa006ae0460d9d2606f62df60fe50d7, disabling compactions & flushes 2024-11-12T10:30:18,575 INFO [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:18,575 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:18,575 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. after waiting 0 ms 2024-11-12T10:30:18,575 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:18,576 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf1/f523ccd65051495cb7b4254bc8d69ac5, hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf1/3fdce4a56f2f487fb98fd3e11ccaef69] to archive 2024-11-12T10:30:18,579 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:30:18,583 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf1/f523ccd65051495cb7b4254bc8d69ac5 to hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf1/f523ccd65051495cb7b4254bc8d69ac5 2024-11-12T10:30:18,585 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf1/3fdce4a56f2f487fb98fd3e11ccaef69 to hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf1/3fdce4a56f2f487fb98fd3e11ccaef69 2024-11-12T10:30:18,598 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf2/d00170be72a44d51b8524ceeb3cd2b84] to archive 2024-11-12T10:30:18,599 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:30:18,601 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf2/d00170be72a44d51b8524ceeb3cd2b84 to hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf2/d00170be72a44d51b8524ceeb3cd2b84 2024-11-12T10:30:18,606 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-12T10:30:18,607 INFO [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:18,607 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for eaa006ae0460d9d2606f62df60fe50d7: Waiting for close lock at 1731407418575Running coprocessor pre-close hooks at 1731407418575Disabling compacts and flushes for region at 1731407418575Disabling writes for close at 1731407418575Writing region close event to WAL at 1731407418602 (+27 ms)Running coprocessor post-close hooks at 1731407418607 (+5 ms)Closed at 1731407418607 2024-11-12T10:30:18,607 INFO [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding eaa006ae0460d9d2606f62df60fe50d7 move to a10fb4296bab,37011,1731407402537 record at close sequenceid=12 2024-11-12T10:30:18,610 INFO [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,611 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=eaa006ae0460d9d2606f62df60fe50d7, regionState=CLOSED 2024-11-12T10:30:18,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure eaa006ae0460d9d2606f62df60fe50d7, server=a10fb4296bab,45327,1731407402438 because future has completed 2024-11-12T10:30:18,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-12T10:30:18,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure eaa006ae0460d9d2606f62df60fe50d7, server=a10fb4296bab,45327,1731407402438 in 195 msec 2024-11-12T10:30:18,619 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, REOPEN/MOVE; state=CLOSED, location=a10fb4296bab,37011,1731407402537; forceNewPlan=false, retain=false 2024-11-12T10:30:18,770 INFO [a10fb4296bab:35883 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-12T10:30:18,770 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=eaa006ae0460d9d2606f62df60fe50d7, regionState=OPENING, regionLocation=a10fb4296bab,37011,1731407402537 2024-11-12T10:30:18,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, REOPEN/MOVE because future has completed 2024-11-12T10:30:18,775 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaa006ae0460d9d2606f62df60fe50d7, server=a10fb4296bab,37011,1731407402537}] 2024-11-12T10:30:18,935 INFO [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:18,936 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => eaa006ae0460d9d2606f62df60fe50d7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:18,936 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,937 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:18,937 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,937 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,940 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,942 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaa006ae0460d9d2606f62df60fe50d7 columnFamilyName cf1 2024-11-12T10:30:18,942 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:18,949 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf1/51d72d268d604d5d8b19f68a4a5b82f8 2024-11-12T10:30:18,949 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(327): Store=eaa006ae0460d9d2606f62df60fe50d7/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:18,949 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,950 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaa006ae0460d9d2606f62df60fe50d7 columnFamilyName cf2 2024-11-12T10:30:18,951 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:18,958 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf2/4f3a8aef788d431399dd31218532ab7c 2024-11-12T10:30:18,958 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(327): Store=eaa006ae0460d9d2606f62df60fe50d7/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:18,958 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,959 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,960 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,961 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,961 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,962 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-12T10:30:18,963 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,964 INFO [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened eaa006ae0460d9d2606f62df60fe50d7; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72811133, jitterRate=0.08497042953968048}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-12T10:30:18,964 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:18,965 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for eaa006ae0460d9d2606f62df60fe50d7: Running coprocessor pre-open hook at 1731407418937Writing region info on filesystem at 1731407418937Initializing all the Stores at 1731407418939 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407418939Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407418940 (+1 ms)Cleaning up temporary data from old regions at 1731407418961 (+21 ms)Running coprocessor post-open hooks at 1731407418964 (+3 ms)Region opened successfully at 1731407418965 (+1 ms) 2024-11-12T10:30:18,966 INFO [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., pid=12, masterSystemTime=1731407418931 2024-11-12T10:30:18,969 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:18,969 INFO [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:18,970 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=eaa006ae0460d9d2606f62df60fe50d7, regionState=OPEN, openSeqNum=18, regionLocation=a10fb4296bab,37011,1731407402537 2024-11-12T10:30:18,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaa006ae0460d9d2606f62df60fe50d7, server=a10fb4296bab,37011,1731407402537 because future has completed 2024-11-12T10:30:18,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-12T10:30:18,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure eaa006ae0460d9d2606f62df60fe50d7, server=a10fb4296bab,37011,1731407402537 in 199 msec 2024-11-12T10:30:18,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, REOPEN/MOVE in 564 msec 2024-11-12T10:30:19,017 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T10:30:19,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T10:30:19,020 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server a10fb4296bab,37011,1731407402537: testing ***** 2024-11-12T10:30:19,021 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-12T10:30:19,023 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-12T10:30:19,024 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-12T10:30:19,027 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-12T10:30:19,028 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-12T10:30:19,036 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 225711616 }, "NonHeapMemoryUsage": { "committed": 170786816, "init": 7667712, "max": -1, "used": 168164312 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "a10fb4296bab", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 9, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 8, "TotalCallTime_95th_percentile": 8, "TotalCallTime_98th_percentile": 8, "TotalCallTime_99th_percentile": 8, "TotalCallTime_99.9th_percentile": 8, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "a10fb4296bab", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:55409", "tag.serverName": "a10fb4296bab,40713,1731407402577", "tag.clusterId": "75eb3984-a818-4a7a-bd42-c0aa0fc13589", "tag.Context": "regionserver", "tag.Hostname": "a10fb4296bab", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731407402577, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-12T10:30:19,040 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35883 {}] master.MasterRpcServices(700): a10fb4296bab,37011,1731407402537 reported a fatal error: ***** ABORTING region server a10fb4296bab,37011,1731407402537: testing ***** 2024-11-12T10:30:19,045 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a10fb4296bab,37011,1731407402537' ***** 2024-11-12T10:30:19,045 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-12T10:30:19,045 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T10:30:19,046 INFO [RS:1;a10fb4296bab:37011 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-12T10:30:19,046 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T10:30:19,046 INFO [RS:1;a10fb4296bab:37011 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-12T10:30:19,046 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.HRegionServer(3091): Received CLOSE for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:19,046 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.HRegionServer(956): aborting server a10fb4296bab,37011,1731407402537 2024-11-12T10:30:19,046 INFO [RS:1;a10fb4296bab:37011 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T10:30:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45327 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:46506 deadline: 1731407479046, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a10fb4296bab port=37011 startCode=1731407402537. As of locationSeqNum=12. 2024-11-12T10:30:19,047 INFO [RS:1;a10fb4296bab:37011 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;a10fb4296bab:37011. 2024-11-12T10:30:19,047 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eaa006ae0460d9d2606f62df60fe50d7, disabling compactions & flushes 2024-11-12T10:30:19,047 INFO [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:19,047 DEBUG [RS:1;a10fb4296bab:37011 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T10:30:19,047 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:19,047 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. after waiting 0 ms 2024-11-12T10:30:19,047 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:19,047 DEBUG [RS:1;a10fb4296bab:37011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:30:19,047 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,45327,1731407402438, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,45327,1731407402438, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a10fb4296bab port=37011 startCode=1731407402537. As of locationSeqNum=12. 2024-11-12T10:30:19,048 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T10:30:19,048 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,45327,1731407402438, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a10fb4296bab port=37011 startCode=1731407402537. As of locationSeqNum=12. 2024-11-12T10:30:19,048 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T10:30:19,048 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T10:30:19,048 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,45327,1731407402438, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a10fb4296bab port=37011 startCode=1731407402537. As of locationSeqNum=12. 2024-11-12T10:30:19,048 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T10:30:19,049 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-12T10:30:19,049 DEBUG [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T10:30:19,049 DEBUG [RS:1;a10fb4296bab:37011 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, eaa006ae0460d9d2606f62df60fe50d7=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.} 2024-11-12T10:30:19,049 INFO [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T10:30:19,049 DEBUG [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T10:30:19,049 DEBUG [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T10:30:19,049 DEBUG [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T10:30:19,050 DEBUG [RS:1;a10fb4296bab:37011 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:19,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a10fb4296bab,37011,1731407402537 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:19,051 INFO [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:19,051 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a10fb4296bab,37011,1731407402537 aborting 2024-11-12T10:30:19,051 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a10fb4296bab,37011,1731407402537 aborting 2024-11-12T10:30:19,051 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eaa006ae0460d9d2606f62df60fe50d7: Waiting for close lock at 1731407419046Running coprocessor pre-close hooks at 1731407419047 (+1 ms)Disabling compacts and flushes for region at 1731407419047Disabling writes for close at 1731407419047Writing region close event to WAL at 1731407419050 (+3 ms)Running coprocessor post-close hooks at 1731407419050Closed at 1731407419051 (+1 ms) 2024-11-12T10:30:19,051 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1 from cache 2024-11-12T10:30:19,051 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:19,052 ERROR [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-12T10:30:19,053 DEBUG [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T10:30:19,053 INFO [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T10:30:19,053 DEBUG [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731407419049Running coprocessor pre-close hooks at 1731407419049Disabling compacts and flushes for region at 1731407419049Disabling writes for close at 1731407419049Writing region close event to WAL at 1731407419052 (+3 ms)Running coprocessor post-close hooks at 1731407419052Closed at 1731407419053 (+1 ms) 2024-11-12T10:30:19,053 DEBUG [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T10:30:19,136 INFO [regionserver/a10fb4296bab:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T10:30:19,152 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:30:19,153 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1] 2024-11-12T10:30:19,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a10fb4296bab,37011,1731407402537 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:19,155 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a10fb4296bab,37011,1731407402537 aborting 2024-11-12T10:30:19,155 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a10fb4296bab,37011,1731407402537 aborting 2024-11-12T10:30:19,155 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1 from cache 2024-11-12T10:30:19,250 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.HRegionServer(976): stopping server a10fb4296bab,37011,1731407402537; all regions closed. 2024-11-12T10:30:19,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741836_1012 (size=6845) 2024-11-12T10:30:19,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741836_1012 (size=6845) 2024-11-12T10:30:19,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741836_1012 (size=6845) 2024-11-12T10:30:19,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741834_1010 (size=2142) 2024-11-12T10:30:19,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741834_1010 (size=2142) 2024-11-12T10:30:19,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741834_1010 (size=2142) 2024-11-12T10:30:19,264 DEBUG [RS:1;a10fb4296bab:37011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:30:19,264 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T10:30:19,264 INFO [RS:1;a10fb4296bab:37011 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T10:30:19,264 INFO [RS:1;a10fb4296bab:37011 {}] hbase.ChoreService(370): Chore service for: regionserver/a10fb4296bab:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-12T10:30:19,264 INFO [RS:1;a10fb4296bab:37011 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T10:30:19,264 INFO [regionserver/a10fb4296bab:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T10:30:19,265 INFO [RS:1;a10fb4296bab:37011 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37011 2024-11-12T10:30:19,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T10:30:19,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37011-0x1012e984b560002, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a10fb4296bab,37011,1731407402537 2024-11-12T10:30:19,334 INFO [RS:1;a10fb4296bab:37011 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T10:30:19,337 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a10fb4296bab,37011,1731407402537] 2024-11-12T10:30:19,355 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a10fb4296bab,37011,1731407402537 already deleted, retry=false 2024-11-12T10:30:19,355 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of a10fb4296bab,37011,1731407402537 on a10fb4296bab,35883,1731407401663 2024-11-12T10:30:19,359 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure a10fb4296bab,37011,1731407402537, splitWal=true, meta=true 2024-11-12T10:30:19,362 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=13 for a10fb4296bab,37011,1731407402537 (carryingMeta=true) a10fb4296bab,37011,1731407402537/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@5e3b687c[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-12T10:30:19,362 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:30:19,362 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure a10fb4296bab,37011,1731407402537, splitWal=true, meta=true 2024-11-12T10:30:19,364 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1] 2024-11-12T10:30:19,364 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure a10fb4296bab,37011,1731407402537, splitWal=true, meta=true, isMeta: true 2024-11-12T10:30:19,365 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server a10fb4296bab:37011 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: a10fb4296bab/172.17.0.3:37011 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:19,366 DEBUG [PEWorker-5 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting 2024-11-12T10:30:19,366 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1, error=java.net.ConnectException: Call to address=a10fb4296bab:37011 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: a10fb4296bab/172.17.0.3:37011 2024-11-12T10:30:19,366 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-12T10:30:19,366 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1 from cache 2024-11-12T10:30:19,367 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address a10fb4296bab:37011 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: a10fb4296bab/172.17.0.3:37011 2024-11-12T10:30:19,367 INFO [PEWorker-5 {}] master.SplitWALManager(105): a10fb4296bab,37011,1731407402537 WAL count=1, meta=true 2024-11-12T10:30:19,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta}] 2024-11-12T10:30:19,376 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=a10fb4296bab,45327,1731407402438 2024-11-12T10:30:19,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta, worker=a10fb4296bab,45327,1731407402438}] 2024-11-12T10:30:19,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37011-0x1012e984b560002, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T10:30:19,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37011-0x1012e984b560002, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T10:30:19,445 INFO [RS:1;a10fb4296bab:37011 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T10:30:19,445 INFO [RS:1;a10fb4296bab:37011 {}] regionserver.HRegionServer(1031): Exiting; stopping=a10fb4296bab,37011,1731407402537; zookeeper connection closed. 2024-11-12T10:30:19,446 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4c78297a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4c78297a 2024-11-12T10:30:19,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45327 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-12T10:30:19,557 INFO [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta, size=6.7 K (6845bytes) 2024-11-12T10:30:19,557 INFO [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta 2024-11-12T10:30:19,558 INFO [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta after 0ms 2024-11-12T10:30:19,560 INFO [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta took 3ms 2024-11-12T10:30:19,568 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-12T10:30:19,568 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta so closing down 2024-11-12T10:30:19,568 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T10:30:19,570 INFO [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta.temp 2024-11-12T10:30:19,571 INFO [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta.temp 2024-11-12T10:30:19,571 INFO [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T10:30:19,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741892_1070 (size=6604) 2024-11-12T10:30:19,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741892_1070 (size=6604) 2024-11-12T10:30:19,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741892_1070 (size=6604) 2024-11-12T10:30:19,581 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-12T10:30:19,582 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta.temp to hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-12T10:30:19,583 INFO [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 21 ms; skipped=1; WAL=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false 2024-11-12T10:30:19,583 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta, journal: Splitting hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta, size=6.7 K (6845bytes) at 1731407419557Finishing writing output for hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta so closing down at 1731407419568 (+11 ms)Creating recovered edits writer path=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta.temp at 1731407419571 (+3 ms)3 split writer threads finished at 1731407419571Closed recovered edits writer path=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731407419581 (+10 ms)Rename recovered edits hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta.temp to hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1731407419582 (+1 ms)Processed 16 edits across 1 Regions in 21 ms; skipped=1; WAL=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false at 1731407419583 (+1 ms) 2024-11-12T10:30:19,583 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta 2024-11-12T10:30:19,584 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-12T10:30:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35883 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-12T10:30:19,591 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta to hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/oldWALs 2024-11-12T10:30:19,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-12T10:30:19,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta, worker=a10fb4296bab,45327,1731407402438 in 213 msec 2024-11-12T10:30:19,595 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=a10fb4296bab,45327,1731407402438 2024-11-12T10:30:19,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-12T10:30:19,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure a10fb4296bab%2C37011%2C1731407402537.meta.1731407404507.meta, worker=a10fb4296bab,45327,1731407402438 in 227 msec 2024-11-12T10:30:19,601 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting dir is empty, no logs to split. 2024-11-12T10:30:19,601 INFO [PEWorker-5 {}] master.SplitWALManager(105): a10fb4296bab,37011,1731407402537 WAL count=0, meta=true 2024-11-12T10:30:19,601 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if a10fb4296bab,37011,1731407402537 WAL splitting is done? wals=0, meta=true 2024-11-12T10:30:19,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T10:30:19,605 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T10:30:19,606 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T10:30:19,672 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:30:19,674 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1] 2024-11-12T10:30:19,674 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to a10fb4296bab:37011 this server is in the failed servers list 2024-11-12T10:30:19,675 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=a10fb4296bab:37011 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a10fb4296bab:37011 2024-11-12T10:30:19,675 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a10fb4296bab:37011 2024-11-12T10:30:19,675 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1 from cache 2024-11-12T10:30:19,757 DEBUG [a10fb4296bab:35883 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-12T10:30:19,757 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(204): Hosts are {a10fb4296bab=0} racks are {/default-rack=0} 2024-11-12T10:30:19,757 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T10:30:19,757 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T10:30:19,757 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T10:30:19,757 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T10:30:19,757 INFO [a10fb4296bab:35883 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T10:30:19,757 INFO [a10fb4296bab:35883 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T10:30:19,757 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T10:30:19,758 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a10fb4296bab,45327,1731407402438 2024-11-12T10:30:19,767 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a10fb4296bab,45327,1731407402438, state=OPENING 2024-11-12T10:30:19,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T10:30:19,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T10:30:19,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T10:30:19,826 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T10:30:19,826 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T10:30:19,826 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T10:30:19,826 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T10:30:19,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a10fb4296bab,45327,1731407402438}] 2024-11-12T10:30:19,993 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T10:30:19,994 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:19,994 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T10:30:19,996 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a10fb4296bab%2C45327%2C1731407402438.meta, suffix=.meta, logDir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,45327,1731407402438, archiveDir=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/oldWALs, maxLogs=32 2024-11-12T10:30:20,007 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,45327,1731407402438/a10fb4296bab%2C45327%2C1731407402438.meta.1731407419996.meta, exclude list is [], retry=0 2024-11-12T10:30:20,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:20,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:20,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:20,012 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,45327,1731407402438/a10fb4296bab%2C45327%2C1731407402438.meta.1731407419996.meta 2024-11-12T10:30:20,012 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:33367:33367)] 2024-11-12T10:30:20,013 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:20,013 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T10:30:20,013 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T10:30:20,013 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T10:30:20,013 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T10:30:20,013 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:20,013 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T10:30:20,013 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T10:30:20,015 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T10:30:20,016 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T10:30:20,016 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:20,016 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:30:20,016 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T10:30:20,017 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T10:30:20,017 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:20,017 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:30:20,018 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T10:30:20,018 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T10:30:20,018 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:20,019 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:30:20,019 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T10:30:20,020 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T10:30:20,020 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:20,020 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T10:30:20,020 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T10:30:20,021 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740 2024-11-12T10:30:20,023 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740 2024-11-12T10:30:20,023 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-12T10:30:20,027 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-12T10:30:20,027 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-12T10:30:20,045 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/.tmp/info/985402ad5bfb468e93f3b2957cded7d9 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7./info:regioninfo/1731407418970/Put/seqid=0 2024-11-12T10:30:20,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741894_1072 (size=11177) 2024-11-12T10:30:20,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741894_1072 (size=11177) 2024-11-12T10:30:20,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741894_1072 (size=11177) 2024-11-12T10:30:20,052 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/.tmp/info/985402ad5bfb468e93f3b2957cded7d9 2024-11-12T10:30:20,070 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/.tmp/ns/38f9f4583c4f41968b9021a74f0faf4b is 43, key is default/ns:d/1731407404654/Put/seqid=0 2024-11-12T10:30:20,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741895_1073 (size=5153) 2024-11-12T10:30:20,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741895_1073 (size=5153) 2024-11-12T10:30:20,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741895_1073 (size=5153) 2024-11-12T10:30:20,077 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/.tmp/ns/38f9f4583c4f41968b9021a74f0faf4b 2024-11-12T10:30:20,095 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/.tmp/table/a08be737f08649f8a90cf9933e8dfcb2 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731407417177/Put/seqid=0 2024-11-12T10:30:20,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741896_1074 (size=5431) 2024-11-12T10:30:20,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741896_1074 (size=5431) 2024-11-12T10:30:20,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741896_1074 (size=5431) 2024-11-12T10:30:20,102 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/.tmp/table/a08be737f08649f8a90cf9933e8dfcb2 2024-11-12T10:30:20,108 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/.tmp/info/985402ad5bfb468e93f3b2957cded7d9 as hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/info/985402ad5bfb468e93f3b2957cded7d9 2024-11-12T10:30:20,114 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/info/985402ad5bfb468e93f3b2957cded7d9, entries=36, sequenceid=18, filesize=10.9 K 2024-11-12T10:30:20,115 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/.tmp/ns/38f9f4583c4f41968b9021a74f0faf4b as hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/ns/38f9f4583c4f41968b9021a74f0faf4b 2024-11-12T10:30:20,120 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/ns/38f9f4583c4f41968b9021a74f0faf4b, entries=2, sequenceid=18, filesize=5.0 K 2024-11-12T10:30:20,121 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/.tmp/table/a08be737f08649f8a90cf9933e8dfcb2 as hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/table/a08be737f08649f8a90cf9933e8dfcb2 2024-11-12T10:30:20,126 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/table/a08be737f08649f8a90cf9933e8dfcb2, entries=2, sequenceid=18, filesize=5.3 K 2024-11-12T10:30:20,127 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 100ms, sequenceid=18, compaction requested=false; wal=null 2024-11-12T10:30:20,127 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-12T10:30:20,129 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T10:30:20,129 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T10:30:20,129 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T10:30:20,131 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T10:30:20,133 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-12T10:30:20,134 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60793361, jitterRate=-0.09410832822322845}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T10:30:20,134 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T10:30:20,135 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731407420013Writing region info on filesystem at 1731407420014 (+1 ms)Initializing all the Stores at 1731407420014Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731407420014Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731407420015 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407420015Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731407420015Obtaining lock to block concurrent updates at 1731407420027 (+12 ms)Preparing flush snapshotting stores in 1588230740 at 1731407420027Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1731407420027Flushing stores of hbase:meta,,1.1588230740 at 1731407420027Flushing 1588230740/info: creating writer at 1731407420028 (+1 ms)Flushing 1588230740/info: appending metadata at 1731407420045 (+17 ms)Flushing 1588230740/info: closing flushed file at 1731407420045Flushing 1588230740/ns: creating writer at 1731407420058 (+13 ms)Flushing 1588230740/ns: appending metadata at 1731407420070 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1731407420070Flushing 1588230740/table: creating writer at 1731407420083 (+13 ms)Flushing 1588230740/table: appending metadata at 1731407420095 (+12 ms)Flushing 1588230740/table: closing flushed file at 1731407420095Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f9ca834: reopening flushed file at 1731407420107 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18b8a9c: reopening flushed file at 1731407420114 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50c41bfd: reopening flushed file at 1731407420120 (+6 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 100ms, sequenceid=18, compaction requested=false; wal=null at 1731407420127 (+7 ms)Cleaning up temporary data from old regions at 1731407420129 (+2 ms)Running coprocessor post-open hooks at 1731407420134 (+5 ms)Region opened successfully at 1731407420135 (+1 ms) 2024-11-12T10:30:20,136 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1731407419983 2024-11-12T10:30:20,138 DEBUG [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T10:30:20,138 INFO [RS_OPEN_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T10:30:20,139 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=a10fb4296bab,45327,1731407402438 2024-11-12T10:30:20,140 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a10fb4296bab,45327,1731407402438, state=OPEN 2024-11-12T10:30:20,142 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35883 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=a10fb4296bab,45327,1731407402438, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T10:30:20,183 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:30:20,184 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,45327,1731407402438, seqNum=-1] 2024-11-12T10:30:20,244 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35883 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=a10fb4296bab,45327,1731407402438, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T10:30:20,303 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to a10fb4296bab:37011 this server is in the failed servers list 2024-11-12T10:30:20,303 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=a10fb4296bab:37011 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a10fb4296bab:37011 2024-11-12T10:30:20,303 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a10fb4296bab:37011 2024-11-12T10:30:20,304 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=12 from cache 2024-11-12T10:30:20,349 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35883 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=a10fb4296bab,45327,1731407402438, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T10:30:20,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T10:30:20,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T10:30:20,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T10:30:20,384 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a10fb4296bab,45327,1731407402438 2024-11-12T10:30:20,385 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T10:30:20,385 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T10:30:20,385 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T10:30:20,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-12T10:30:20,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a10fb4296bab,45327,1731407402438 in 559 msec 2024-11-12T10:30:20,390 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-12T10:30:20,390 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 783 msec 2024-11-12T10:30:20,390 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): a10fb4296bab,37011,1731407402537 had 2 regions 2024-11-12T10:30:20,391 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure a10fb4296bab,37011,1731407402537, splitWal=true, meta=true, isMeta: false 2024-11-12T10:30:20,394 INFO [PEWorker-2 {}] master.SplitWALManager(105): a10fb4296bab,37011,1731407402537 WAL count=1, meta=false 2024-11-12T10:30:20,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure a10fb4296bab%2C37011%2C1731407402537.1731407404072}] 2024-11-12T10:30:20,396 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=a10fb4296bab,45327,1731407402438 2024-11-12T10:30:20,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure a10fb4296bab%2C37011%2C1731407402537.1731407404072, worker=a10fb4296bab,45327,1731407402438}] 2024-11-12T10:30:20,522 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=18] 2024-11-12T10:30:20,522 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to a10fb4296bab:37011 this server is in the failed servers list 2024-11-12T10:30:20,522 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=a10fb4296bab:37011 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a10fb4296bab:37011 2024-11-12T10:30:20,523 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a10fb4296bab:37011 2024-11-12T10:30:20,523 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=18 from cache 2024-11-12T10:30:20,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45327 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-12T10:30:20,568 INFO [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.1731407404072, size=2.1 K (2142bytes) 2024-11-12T10:30:20,568 INFO [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.1731407404072 2024-11-12T10:30:20,569 INFO [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.1731407404072 after 0ms 2024-11-12T10:30:20,571 INFO [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.1731407404072 took 3ms 2024-11-12T10:30:20,573 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for eaa006ae0460d9d2606f62df60fe50d7: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-12T10:30:20,573 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.1731407404072 so closing down 2024-11-12T10:30:20,574 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T10:30:20,574 INFO [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T10:30:20,574 INFO [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.1731407404072, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-11-12T10:30:20,574 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.1731407404072, journal: Splitting hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.1731407404072, size=2.1 K (2142bytes) at 1731407420568Finishing writing output for hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.1731407404072 so closing down at 1731407420574 (+6 ms)3 split writer threads finished at 1731407420574Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.1731407404072, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1731407420574 2024-11-12T10:30:20,574 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.1731407404072 2024-11-12T10:30:20,574 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a10fb4296bab:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-12T10:30:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35883 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-12T10:30:20,578 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting/a10fb4296bab%2C37011%2C1731407402537.1731407404072 to hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/oldWALs 2024-11-12T10:30:20,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-12T10:30:20,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a10fb4296bab%2C37011%2C1731407402537.1731407404072, worker=a10fb4296bab,45327,1731407402438 in 182 msec 2024-11-12T10:30:20,582 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=a10fb4296bab,45327,1731407402438 2024-11-12T10:30:20,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-12T10:30:20,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure a10fb4296bab%2C37011%2C1731407402537.1731407404072, worker=a10fb4296bab,45327,1731407402438 in 189 msec 2024-11-12T10:30:20,587 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,37011,1731407402537-splitting dir is empty, no logs to split. 2024-11-12T10:30:20,587 INFO [PEWorker-2 {}] master.SplitWALManager(105): a10fb4296bab,37011,1731407402537 WAL count=0, meta=false 2024-11-12T10:30:20,587 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if a10fb4296bab,37011,1731407402537 WAL splitting is done? wals=0, meta=false 2024-11-12T10:30:20,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, ASSIGN}] 2024-11-12T10:30:20,590 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, ASSIGN 2024-11-12T10:30:20,591 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T10:30:20,742 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(204): Hosts are {a10fb4296bab=0} racks are {/default-rack=0} 2024-11-12T10:30:20,742 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T10:30:20,742 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T10:30:20,742 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T10:30:20,742 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T10:30:20,742 INFO [a10fb4296bab:35883 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T10:30:20,743 INFO [a10fb4296bab:35883 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T10:30:20,743 DEBUG [a10fb4296bab:35883 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T10:30:20,743 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=eaa006ae0460d9d2606f62df60fe50d7, regionState=OPENING, regionLocation=a10fb4296bab,45327,1731407402438 2024-11-12T10:30:20,745 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server a10fb4296bab:37011 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a10fb4296bab/172.17.0.3:37011 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:20,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1, error=java.net.ConnectException: Call to address=a10fb4296bab:37011 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a10fb4296bab/172.17.0.3:37011 2024-11-12T10:30:20,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T10:30:20,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,37011,1731407402537, seqNum=-1 from cache 2024-11-12T10:30:20,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address a10fb4296bab:37011 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a10fb4296bab/172.17.0.3:37011 2024-11-12T10:30:20,838 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=18] 2024-11-12T10:30:20,839 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to a10fb4296bab:37011 this server is in the failed servers list 2024-11-12T10:30:20,839 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=a10fb4296bab:37011 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a10fb4296bab:37011 2024-11-12T10:30:20,839 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a10fb4296bab:37011 2024-11-12T10:30:20,840 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,37011,1731407402537, seqNum=18 from cache 2024-11-12T10:30:20,852 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T10:30:20,852 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a10fb4296bab,45327,1731407402438, seqNum=-1] 2024-11-12T10:30:20,852 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T10:30:20,853 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48083, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T10:30:20,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, ASSIGN because future has completed 2024-11-12T10:30:20,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaa006ae0460d9d2606f62df60fe50d7, server=a10fb4296bab,45327,1731407402438}] 2024-11-12T10:30:21,012 INFO [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:21,013 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => eaa006ae0460d9d2606f62df60fe50d7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:21,013 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:21,013 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:21,013 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:21,013 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:21,015 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:21,015 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaa006ae0460d9d2606f62df60fe50d7 columnFamilyName cf1 2024-11-12T10:30:21,016 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:21,022 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf1/51d72d268d604d5d8b19f68a4a5b82f8 2024-11-12T10:30:21,022 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(327): Store=eaa006ae0460d9d2606f62df60fe50d7/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:21,022 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:21,023 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaa006ae0460d9d2606f62df60fe50d7 columnFamilyName cf2 2024-11-12T10:30:21,023 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:21,029 DEBUG [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/cf2/4f3a8aef788d431399dd31218532ab7c 2024-11-12T10:30:21,030 INFO [StoreOpener-eaa006ae0460d9d2606f62df60fe50d7-1 {}] regionserver.HStore(327): Store=eaa006ae0460d9d2606f62df60fe50d7/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:21,030 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:21,031 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:21,032 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:21,033 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:21,033 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:21,033 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-12T10:30:21,035 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:21,035 INFO [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened eaa006ae0460d9d2606f62df60fe50d7; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66118325, jitterRate=-0.014760181307792664}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-12T10:30:21,035 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:21,036 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for eaa006ae0460d9d2606f62df60fe50d7: Running coprocessor pre-open hook at 1731407421013Writing region info on filesystem at 1731407421013Initializing all the Stores at 1731407421014 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407421014Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407421014Cleaning up temporary data from old regions at 1731407421033 (+19 ms)Running coprocessor post-open hooks at 1731407421036 (+3 ms)Region opened successfully at 1731407421036 2024-11-12T10:30:21,037 INFO [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., pid=21, masterSystemTime=1731407421009 2024-11-12T10:30:21,039 DEBUG [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:21,039 INFO [RS_OPEN_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:21,040 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=eaa006ae0460d9d2606f62df60fe50d7, regionState=OPEN, openSeqNum=18, regionLocation=a10fb4296bab,45327,1731407402438 2024-11-12T10:30:21,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaa006ae0460d9d2606f62df60fe50d7, server=a10fb4296bab,45327,1731407402438 because future has completed 2024-11-12T10:30:21,046 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-12T10:30:21,047 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure eaa006ae0460d9d2606f62df60fe50d7, server=a10fb4296bab,45327,1731407402438 in 188 msec 2024-11-12T10:30:21,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-12T10:30:21,049 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server a10fb4296bab,37011,1731407402537 after splitting done 2024-11-12T10:30:21,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=eaa006ae0460d9d2606f62df60fe50d7, ASSIGN in 457 msec 2024-11-12T10:30:21,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure a10fb4296bab,37011,1731407402537, splitWal=true, meta=true in 1.6930 sec 2024-11-12T10:30:21,357 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7., hostname=a10fb4296bab,45327,1731407402438, seqNum=18] 2024-11-12T10:30:21,374 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=404 (was 411), OpenFileDescriptor=1042 (was 1022) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=676 (was 735), ProcessCount=11 (was 11), AvailableMemoryMB=7279 (was 7424) 2024-11-12T10:30:21,376 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1042 is superior to 1024 2024-11-12T10:30:21,387 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=404, OpenFileDescriptor=1042, MaxFileDescriptor=1048576, SystemLoadAverage=676, ProcessCount=11, AvailableMemoryMB=7278 2024-11-12T10:30:21,387 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1042 is superior to 1024 2024-11-12T10:30:21,401 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:21,402 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:21,403 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T10:30:21,405 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-36093590, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/hregion-36093590, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:21,417 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-36093590/hregion-36093590.1731407421406, exclude list is [], retry=0 2024-11-12T10:30:21,419 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:21,419 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:21,420 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:21,422 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-36093590/hregion-36093590.1731407421406 2024-11-12T10:30:21,422 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33367:33367)] 2024-11-12T10:30:21,422 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7c6744271f57ac512c61b9d79e7f251f, NAME => 'testReplayEditsWrittenViaHRegion,,1731407421401.7c6744271f57ac512c61b9d79e7f251f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38657/hbase 2024-11-12T10:30:21,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741898_1076 (size=67) 2024-11-12T10:30:21,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741898_1076 (size=67) 2024-11-12T10:30:21,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741898_1076 (size=67) 2024-11-12T10:30:21,440 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731407421401.7c6744271f57ac512c61b9d79e7f251f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:21,441 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,443 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c6744271f57ac512c61b9d79e7f251f columnFamilyName a 2024-11-12T10:30:21,443 DEBUG [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:21,443 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.HStore(327): Store=7c6744271f57ac512c61b9d79e7f251f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:21,444 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,445 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c6744271f57ac512c61b9d79e7f251f columnFamilyName b 2024-11-12T10:30:21,445 DEBUG [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:21,445 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.HStore(327): Store=7c6744271f57ac512c61b9d79e7f251f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:21,445 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,447 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c6744271f57ac512c61b9d79e7f251f columnFamilyName c 2024-11-12T10:30:21,447 DEBUG [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:21,447 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.HStore(327): Store=7c6744271f57ac512c61b9d79e7f251f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:21,447 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,448 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,449 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,450 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,450 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,450 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T10:30:21,451 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,454 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:30:21,454 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7c6744271f57ac512c61b9d79e7f251f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62142092, jitterRate=-0.07401067018508911}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T10:30:21,455 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7c6744271f57ac512c61b9d79e7f251f: Writing region info on filesystem at 1731407421440Initializing all the Stores at 1731407421441 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407421441Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407421441Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407421441Cleaning up temporary data from old regions at 1731407421450 (+9 ms)Region opened successfully at 1731407421455 (+5 ms) 2024-11-12T10:30:21,455 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7c6744271f57ac512c61b9d79e7f251f, disabling compactions & flushes 2024-11-12T10:30:21,455 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731407421401.7c6744271f57ac512c61b9d79e7f251f. 2024-11-12T10:30:21,455 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731407421401.7c6744271f57ac512c61b9d79e7f251f. 2024-11-12T10:30:21,456 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731407421401.7c6744271f57ac512c61b9d79e7f251f. after waiting 0 ms 2024-11-12T10:30:21,456 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731407421401.7c6744271f57ac512c61b9d79e7f251f. 2024-11-12T10:30:21,456 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731407421401.7c6744271f57ac512c61b9d79e7f251f. 2024-11-12T10:30:21,456 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7c6744271f57ac512c61b9d79e7f251f: Waiting for close lock at 1731407421455Disabling compacts and flushes for region at 1731407421455Disabling writes for close at 1731407421456 (+1 ms)Writing region close event to WAL at 1731407421456Closed at 1731407421456 2024-11-12T10:30:21,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741897_1075 (size=93) 2024-11-12T10:30:21,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741897_1075 (size=93) 2024-11-12T10:30:21,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741897_1075 (size=93) 2024-11-12T10:30:21,464 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T10:30:21,465 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-36093590:(num 1731407421406) 2024-11-12T10:30:21,465 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:21,467 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:21,480 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400/wal.1731407421467, exclude list is [], retry=0 2024-11-12T10:30:21,482 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:21,482 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:21,483 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:21,484 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400/wal.1731407421467 2024-11-12T10:30:21,484 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:39823:39823)] 2024-11-12T10:30:21,484 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7c6744271f57ac512c61b9d79e7f251f, NAME => 'testReplayEditsWrittenViaHRegion,,1731407421401.7c6744271f57ac512c61b9d79e7f251f.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:21,485 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731407421401.7c6744271f57ac512c61b9d79e7f251f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:21,485 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,485 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,486 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,487 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c6744271f57ac512c61b9d79e7f251f columnFamilyName a 2024-11-12T10:30:21,487 DEBUG [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:21,488 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.HStore(327): Store=7c6744271f57ac512c61b9d79e7f251f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:21,488 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,489 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c6744271f57ac512c61b9d79e7f251f columnFamilyName b 2024-11-12T10:30:21,489 DEBUG [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:21,489 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.HStore(327): Store=7c6744271f57ac512c61b9d79e7f251f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:21,489 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,490 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c6744271f57ac512c61b9d79e7f251f columnFamilyName c 2024-11-12T10:30:21,490 DEBUG [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:21,490 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.HStore(327): Store=7c6744271f57ac512c61b9d79e7f251f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:21,491 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,491 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,493 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,494 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,494 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,494 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T10:30:21,496 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,497 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7c6744271f57ac512c61b9d79e7f251f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68365647, jitterRate=0.01872752606868744}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T10:30:21,497 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7c6744271f57ac512c61b9d79e7f251f: Writing region info on filesystem at 1731407421485Initializing all the Stores at 1731407421486 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407421486Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407421486Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407421486Cleaning up temporary data from old regions at 1731407421494 (+8 ms)Region opened successfully at 1731407421497 (+3 ms) 2024-11-12T10:30:21,528 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7c6744271f57ac512c61b9d79e7f251f 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-12T10:30:21,548 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/.tmp/a/7dda32bf55224660a58cae21d8934604 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731407421497/Put/seqid=0 2024-11-12T10:30:21,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741900_1078 (size=5958) 2024-11-12T10:30:21,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741900_1078 (size=5958) 2024-11-12T10:30:21,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741900_1078 (size=5958) 2024-11-12T10:30:21,556 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/.tmp/a/7dda32bf55224660a58cae21d8934604 2024-11-12T10:30:21,576 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/.tmp/b/fed8b7e788a84ba2b4912192fbc45a34 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731407421508/Put/seqid=0 2024-11-12T10:30:21,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741901_1079 (size=5958) 2024-11-12T10:30:21,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741901_1079 (size=5958) 2024-11-12T10:30:21,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741901_1079 (size=5958) 2024-11-12T10:30:21,582 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/.tmp/b/fed8b7e788a84ba2b4912192fbc45a34 2024-11-12T10:30:21,601 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/.tmp/c/ad979b51e29c4e0f8fd3e2966fb0b814 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731407421518/Put/seqid=0 2024-11-12T10:30:21,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741902_1080 (size=5958) 2024-11-12T10:30:21,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741902_1080 (size=5958) 2024-11-12T10:30:21,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741902_1080 (size=5958) 2024-11-12T10:30:21,608 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/.tmp/c/ad979b51e29c4e0f8fd3e2966fb0b814 2024-11-12T10:30:21,613 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/.tmp/a/7dda32bf55224660a58cae21d8934604 as hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/a/7dda32bf55224660a58cae21d8934604 2024-11-12T10:30:21,618 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/a/7dda32bf55224660a58cae21d8934604, entries=10, sequenceid=33, filesize=5.8 K 2024-11-12T10:30:21,619 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/.tmp/b/fed8b7e788a84ba2b4912192fbc45a34 as hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/b/fed8b7e788a84ba2b4912192fbc45a34 2024-11-12T10:30:21,624 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/b/fed8b7e788a84ba2b4912192fbc45a34, entries=10, sequenceid=33, filesize=5.8 K 2024-11-12T10:30:21,625 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/.tmp/c/ad979b51e29c4e0f8fd3e2966fb0b814 as hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/c/ad979b51e29c4e0f8fd3e2966fb0b814 2024-11-12T10:30:21,631 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/c/ad979b51e29c4e0f8fd3e2966fb0b814, entries=10, sequenceid=33, filesize=5.8 K 2024-11-12T10:30:21,632 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 7c6744271f57ac512c61b9d79e7f251f in 104ms, sequenceid=33, compaction requested=false 2024-11-12T10:30:21,632 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7c6744271f57ac512c61b9d79e7f251f: 2024-11-12T10:30:21,632 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7c6744271f57ac512c61b9d79e7f251f, disabling compactions & flushes 2024-11-12T10:30:21,632 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731407421401.7c6744271f57ac512c61b9d79e7f251f. 2024-11-12T10:30:21,632 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731407421401.7c6744271f57ac512c61b9d79e7f251f. 2024-11-12T10:30:21,632 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731407421401.7c6744271f57ac512c61b9d79e7f251f. after waiting 0 ms 2024-11-12T10:30:21,632 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731407421401.7c6744271f57ac512c61b9d79e7f251f. 2024-11-12T10:30:21,634 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731407421401.7c6744271f57ac512c61b9d79e7f251f. 2024-11-12T10:30:21,634 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7c6744271f57ac512c61b9d79e7f251f: Waiting for close lock at 1731407421632Disabling compacts and flushes for region at 1731407421632Disabling writes for close at 1731407421632Writing region close event to WAL at 1731407421633 (+1 ms)Closed at 1731407421633 2024-11-12T10:30:21,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741899_1077 (size=5937) 2024-11-12T10:30:21,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741899_1077 (size=5937) 2024-11-12T10:30:21,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741899_1077 (size=5937) 2024-11-12T10:30:21,642 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/b/fed8b7e788a84ba2b4912192fbc45a34 to hdfs://localhost:38657/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/b/fed8b7e788a84ba2b4912192fbc45a34 2024-11-12T10:30:21,657 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400/wal.1731407421467, size=5.8 K (5937bytes) 2024-11-12T10:30:21,657 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400/wal.1731407421467 2024-11-12T10:30:21,658 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400/wal.1731407421467 after 1ms 2024-11-12T10:30:21,660 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400/wal.1731407421467 took 3ms 2024-11-12T10:30:21,662 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400/wal.1731407421467 so closing down 2024-11-12T10:30:21,663 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T10:30:21,663 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731407421467.temp 2024-11-12T10:30:21,664 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/recovered.edits/0000000000000000003-wal.1731407421467.temp 2024-11-12T10:30:21,665 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T10:30:21,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741903_1081 (size=5223) 2024-11-12T10:30:21,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741903_1081 (size=5223) 2024-11-12T10:30:21,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741903_1081 (size=5223) 2024-11-12T10:30:21,674 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/recovered.edits/0000000000000000003-wal.1731407421467.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-12T10:30:21,675 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/recovered.edits/0000000000000000003-wal.1731407421467.temp to hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/recovered.edits/0000000000000000032 2024-11-12T10:30:21,675 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:38657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400/wal.1731407421467, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-11-12T10:30:21,676 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400/wal.1731407421467, journal: Splitting hdfs://localhost:38657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400/wal.1731407421467, size=5.8 K (5937bytes) at 1731407421657Finishing writing output for hdfs://localhost:38657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400/wal.1731407421467 so closing down at 1731407421662 (+5 ms)Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/recovered.edits/0000000000000000003-wal.1731407421467.temp at 1731407421664 (+2 ms)3 split writer threads finished at 1731407421665 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/recovered.edits/0000000000000000003-wal.1731407421467.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731407421674 (+9 ms)Rename recovered edits hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/recovered.edits/0000000000000000003-wal.1731407421467.temp to hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/recovered.edits/0000000000000000032 at 1731407421675 (+1 ms)Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:38657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400/wal.1731407421467, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1731407421676 (+1 ms) 2024-11-12T10:30:21,677 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400/wal.1731407421467 to hdfs://localhost:38657/hbase/oldWALs/wal.1731407421467 2024-11-12T10:30:21,678 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/recovered.edits/0000000000000000032 2024-11-12T10:30:21,678 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:21,680 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:21,692 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400/wal.1731407421680, exclude list is [], retry=0 2024-11-12T10:30:21,695 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:21,695 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:21,695 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:21,697 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400/wal.1731407421680 2024-11-12T10:30:21,698 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:33367:33367)] 2024-11-12T10:30:21,698 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7c6744271f57ac512c61b9d79e7f251f, NAME => 'testReplayEditsWrittenViaHRegion,,1731407421401.7c6744271f57ac512c61b9d79e7f251f.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:21,698 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731407421401.7c6744271f57ac512c61b9d79e7f251f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:21,698 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,698 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,700 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,701 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c6744271f57ac512c61b9d79e7f251f columnFamilyName a 2024-11-12T10:30:21,701 DEBUG [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:21,707 DEBUG [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/a/7dda32bf55224660a58cae21d8934604 2024-11-12T10:30:21,707 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.HStore(327): Store=7c6744271f57ac512c61b9d79e7f251f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:21,707 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,709 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c6744271f57ac512c61b9d79e7f251f columnFamilyName b 2024-11-12T10:30:21,709 DEBUG [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:21,710 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.HStore(327): Store=7c6744271f57ac512c61b9d79e7f251f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:21,710 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,711 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c6744271f57ac512c61b9d79e7f251f columnFamilyName c 2024-11-12T10:30:21,711 DEBUG [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:21,717 DEBUG [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/c/ad979b51e29c4e0f8fd3e2966fb0b814 2024-11-12T10:30:21,718 INFO [StoreOpener-7c6744271f57ac512c61b9d79e7f251f-1 {}] regionserver.HStore(327): Store=7c6744271f57ac512c61b9d79e7f251f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:21,718 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,718 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,720 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,721 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/recovered.edits/0000000000000000032 2024-11-12T10:30:21,724 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/recovered.edits/0000000000000000032 2024-11-12T10:30:21,724 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7c6744271f57ac512c61b9d79e7f251f 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-12T10:30:21,738 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/.tmp/b/afda231aba74479db9f92ec5f2a23c47 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731407421508/Put/seqid=0 2024-11-12T10:30:21,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741905_1083 (size=5958) 2024-11-12T10:30:21,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741905_1083 (size=5958) 2024-11-12T10:30:21,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741905_1083 (size=5958) 2024-11-12T10:30:21,751 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/.tmp/b/afda231aba74479db9f92ec5f2a23c47 2024-11-12T10:30:21,758 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/.tmp/b/afda231aba74479db9f92ec5f2a23c47 as hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/b/afda231aba74479db9f92ec5f2a23c47 2024-11-12T10:30:21,765 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/b/afda231aba74479db9f92ec5f2a23c47, entries=10, sequenceid=32, filesize=5.8 K 2024-11-12T10:30:21,766 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 7c6744271f57ac512c61b9d79e7f251f in 42ms, sequenceid=32, compaction requested=false; wal=null 2024-11-12T10:30:21,767 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/recovered.edits/0000000000000000032 2024-11-12T10:30:21,768 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,768 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,769 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T10:30:21,771 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7c6744271f57ac512c61b9d79e7f251f 2024-11-12T10:30:21,774 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/7c6744271f57ac512c61b9d79e7f251f/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-12T10:30:21,775 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7c6744271f57ac512c61b9d79e7f251f; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72723271, jitterRate=0.08366118371486664}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T10:30:21,776 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7c6744271f57ac512c61b9d79e7f251f: Writing region info on filesystem at 1731407421699Initializing all the Stores at 1731407421699Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407421700 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407421700Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407421700Obtaining lock to block concurrent updates at 1731407421724 (+24 ms)Preparing flush snapshotting stores in 7c6744271f57ac512c61b9d79e7f251f at 1731407421724Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731407421401.7c6744271f57ac512c61b9d79e7f251f., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731407421725 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1731407421401.7c6744271f57ac512c61b9d79e7f251f. at 1731407421725Flushing 7c6744271f57ac512c61b9d79e7f251f/b: creating writer at 1731407421725Flushing 7c6744271f57ac512c61b9d79e7f251f/b: appending metadata at 1731407421738 (+13 ms)Flushing 7c6744271f57ac512c61b9d79e7f251f/b: closing flushed file at 1731407421738Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@547b9d68: reopening flushed file at 1731407421757 (+19 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 7c6744271f57ac512c61b9d79e7f251f in 42ms, sequenceid=32, compaction requested=false; wal=null at 1731407421766 (+9 ms)Cleaning up temporary data from old regions at 1731407421768 (+2 ms)Region opened successfully at 1731407421776 (+8 ms) 2024-11-12T10:30:21,804 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=413 (was 404) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:46946 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-663294313-172.17.0.3-1731407397077:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:49818 [Waiting for operation #47] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:47046 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-663294313-172.17.0.3-1731407397077:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:46796 [Waiting for operation #40] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-663294313-172.17.0.3-1731407397077:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:47244 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:50006 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1118 (was 1042) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=676 (was 676), ProcessCount=11 (was 11), AvailableMemoryMB=7254 (was 7278) 2024-11-12T10:30:21,804 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1118 is superior to 1024 2024-11-12T10:30:21,816 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=413, OpenFileDescriptor=1118, MaxFileDescriptor=1048576, SystemLoadAverage=676, ProcessCount=11, AvailableMemoryMB=7254 2024-11-12T10:30:21,816 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1118 is superior to 1024 2024-11-12T10:30:21,830 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:21,832 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:21,833 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T10:30:21,835 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23937813, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/hregion-23937813, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:21,847 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23937813/hregion-23937813.1731407421835, exclude list is [], retry=0 2024-11-12T10:30:21,850 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:21,850 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:21,850 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:21,852 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23937813/hregion-23937813.1731407421835 2024-11-12T10:30:21,852 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:33413:33413)] 2024-11-12T10:30:21,853 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 97a2e4b249b2d44cb1d98a1ca3ee3ae1, NAME => 'testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38657/hbase 2024-11-12T10:30:21,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741907_1085 (size=68) 2024-11-12T10:30:21,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741907_1085 (size=68) 2024-11-12T10:30:21,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741907_1085 (size=68) 2024-11-12T10:30:21,861 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:21,862 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,863 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97a2e4b249b2d44cb1d98a1ca3ee3ae1 columnFamilyName a 2024-11-12T10:30:21,863 DEBUG [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:21,864 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] regionserver.HStore(327): Store=97a2e4b249b2d44cb1d98a1ca3ee3ae1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:21,864 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,865 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97a2e4b249b2d44cb1d98a1ca3ee3ae1 columnFamilyName b 2024-11-12T10:30:21,865 DEBUG [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:21,865 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] regionserver.HStore(327): Store=97a2e4b249b2d44cb1d98a1ca3ee3ae1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:21,865 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,867 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97a2e4b249b2d44cb1d98a1ca3ee3ae1 columnFamilyName c 2024-11-12T10:30:21,867 DEBUG [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:21,867 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] regionserver.HStore(327): Store=97a2e4b249b2d44cb1d98a1ca3ee3ae1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:21,867 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,868 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,868 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,869 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,869 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,870 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T10:30:21,871 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,873 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:30:21,873 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 97a2e4b249b2d44cb1d98a1ca3ee3ae1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62617015, jitterRate=-0.0669337660074234}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T10:30:21,874 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 97a2e4b249b2d44cb1d98a1ca3ee3ae1: Writing region info on filesystem at 1731407421861Initializing all the Stores at 1731407421862 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407421862Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407421862Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407421862Cleaning up temporary data from old regions at 1731407421869 (+7 ms)Region opened successfully at 1731407421874 (+5 ms) 2024-11-12T10:30:21,874 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 97a2e4b249b2d44cb1d98a1ca3ee3ae1, disabling compactions & flushes 2024-11-12T10:30:21,874 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1. 2024-11-12T10:30:21,874 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1. 2024-11-12T10:30:21,874 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1. after waiting 0 ms 2024-11-12T10:30:21,874 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1. 2024-11-12T10:30:21,874 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1. 2024-11-12T10:30:21,875 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 97a2e4b249b2d44cb1d98a1ca3ee3ae1: Waiting for close lock at 1731407421874Disabling compacts and flushes for region at 1731407421874Disabling writes for close at 1731407421874Writing region close event to WAL at 1731407421874Closed at 1731407421874 2024-11-12T10:30:21,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741906_1084 (size=93) 2024-11-12T10:30:21,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741906_1084 (size=93) 2024-11-12T10:30:21,880 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T10:30:21,880 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23937813:(num 1731407421835) 2024-11-12T10:30:21,880 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:21,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741906_1084 (size=93) 2024-11-12T10:30:21,882 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:21,894 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829/wal.1731407421882, exclude list is [], retry=0 2024-11-12T10:30:21,897 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:21,897 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:21,898 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:21,899 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829/wal.1731407421882 2024-11-12T10:30:21,899 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:39823:39823)] 2024-11-12T10:30:21,959 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 97a2e4b249b2d44cb1d98a1ca3ee3ae1, NAME => 'testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:21,961 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,961 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:21,961 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,961 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,963 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,964 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97a2e4b249b2d44cb1d98a1ca3ee3ae1 columnFamilyName a 2024-11-12T10:30:21,964 DEBUG [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:21,965 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] regionserver.HStore(327): Store=97a2e4b249b2d44cb1d98a1ca3ee3ae1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:21,965 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,966 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97a2e4b249b2d44cb1d98a1ca3ee3ae1 columnFamilyName b 2024-11-12T10:30:21,966 DEBUG [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:21,967 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] regionserver.HStore(327): Store=97a2e4b249b2d44cb1d98a1ca3ee3ae1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:21,967 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,968 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97a2e4b249b2d44cb1d98a1ca3ee3ae1 columnFamilyName c 2024-11-12T10:30:21,968 DEBUG [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:21,968 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] regionserver.HStore(327): Store=97a2e4b249b2d44cb1d98a1ca3ee3ae1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:21,969 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,969 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,970 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,971 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,972 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,972 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T10:30:21,973 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,974 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 97a2e4b249b2d44cb1d98a1ca3ee3ae1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65399085, jitterRate=-0.025477692484855652}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T10:30:21,974 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:21,975 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 97a2e4b249b2d44cb1d98a1ca3ee3ae1: Running coprocessor pre-open hook at 1731407421961Writing region info on filesystem at 1731407421961Initializing all the Stores at 1731407421962 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407421962Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407421963 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407421963Cleaning up temporary data from old regions at 1731407421972 (+9 ms)Running coprocessor post-open hooks at 1731407421974 (+2 ms)Region opened successfully at 1731407421975 (+1 ms) 2024-11-12T10:30:21,986 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 97a2e4b249b2d44cb1d98a1ca3ee3ae1 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-12T10:30:21,987 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 97a2e4b249b2d44cb1d98a1ca3ee3ae1/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:22,005 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-12T10:30:22,006 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-12T10:30:22,007 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-12T10:30:22,007 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-12T10:30:22,008 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-12T10:30:22,008 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-12T10:30:22,009 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-12T10:30:22,009 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-12T10:30:22,010 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-12T10:30:22,011 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-12T10:30:22,673 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T10:30:22,988 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 97a2e4b249b2d44cb1d98a1ca3ee3ae1/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:23,990 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 97a2e4b249b2d44cb1d98a1ca3ee3ae1/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:24,991 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 97a2e4b249b2d44cb1d98a1ca3ee3ae1/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:25,993 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 97a2e4b249b2d44cb1d98a1ca3ee3ae1/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:26,994 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 97a2e4b249b2d44cb1d98a1ca3ee3ae1/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:27,995 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 97a2e4b249b2d44cb1d98a1ca3ee3ae1/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:28,996 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 97a2e4b249b2d44cb1d98a1ca3ee3ae1/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:29,997 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 97a2e4b249b2d44cb1d98a1ca3ee3ae1/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:30,827 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T10:30:30,998 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 97a2e4b249b2d44cb1d98a1ca3ee3ae1/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:31,000 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 97a2e4b249b2d44cb1d98a1ca3ee3ae1: 2024-11-12T10:30:31,000 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1. 2024-11-12T10:30:31,014 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 97a2e4b249b2d44cb1d98a1ca3ee3ae1: 2024-11-12T10:30:31,014 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-12T10:30:31,014 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 97a2e4b249b2d44cb1d98a1ca3ee3ae1, disabling compactions & flushes 2024-11-12T10:30:31,014 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1. 2024-11-12T10:30:31,014 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1. 2024-11-12T10:30:31,014 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1. after waiting 0 ms 2024-11-12T10:30:31,014 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1. 2024-11-12T10:30:31,015 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1. 2024-11-12T10:30:31,015 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1. 2024-11-12T10:30:31,015 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 97a2e4b249b2d44cb1d98a1ca3ee3ae1: Waiting for close lock at 1731407431014Running coprocessor pre-close hooks at 1731407431014Disabling compacts and flushes for region at 1731407431014Disabling writes for close at 1731407431014Writing region close event to WAL at 1731407431015 (+1 ms)Running coprocessor post-close hooks at 1731407431015Closed at 1731407431015 2024-11-12T10:30:31,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741908_1086 (size=3941) 2024-11-12T10:30:31,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741908_1086 (size=3941) 2024-11-12T10:30:31,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741908_1086 (size=3941) 2024-11-12T10:30:31,031 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829/wal.1731407421882, size=3.8 K (3941bytes) 2024-11-12T10:30:31,031 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829/wal.1731407421882 2024-11-12T10:30:31,032 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829/wal.1731407421882 after 1ms 2024-11-12T10:30:31,034 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829/wal.1731407421882 took 3ms 2024-11-12T10:30:31,036 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829/wal.1731407421882 so closing down 2024-11-12T10:30:31,036 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T10:30:31,037 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731407421882.temp 2024-11-12T10:30:31,038 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/recovered.edits/0000000000000000004-wal.1731407421882.temp 2024-11-12T10:30:31,039 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T10:30:31,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741909_1087 (size=2983) 2024-11-12T10:30:31,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741909_1087 (size=2983) 2024-11-12T10:30:31,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741909_1087 (size=2983) 2024-11-12T10:30:31,045 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/recovered.edits/0000000000000000004-wal.1731407421882.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-12T10:30:31,047 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/recovered.edits/0000000000000000004-wal.1731407421882.temp to hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/recovered.edits/0000000000000000026 2024-11-12T10:30:31,047 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:38657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829/wal.1731407421882, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-11-12T10:30:31,047 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829/wal.1731407421882, journal: Splitting hdfs://localhost:38657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829/wal.1731407421882, size=3.8 K (3941bytes) at 1731407431031Finishing writing output for hdfs://localhost:38657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829/wal.1731407421882 so closing down at 1731407431036 (+5 ms)Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/recovered.edits/0000000000000000004-wal.1731407421882.temp at 1731407431038 (+2 ms)3 split writer threads finished at 1731407431039 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/recovered.edits/0000000000000000004-wal.1731407421882.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731407431045 (+6 ms)Rename recovered edits hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/recovered.edits/0000000000000000004-wal.1731407421882.temp to hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/recovered.edits/0000000000000000026 at 1731407431047 (+2 ms)Processed 23 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:38657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829/wal.1731407421882, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1731407431047 2024-11-12T10:30:31,048 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829/wal.1731407421882 to hdfs://localhost:38657/hbase/oldWALs/wal.1731407421882 2024-11-12T10:30:31,049 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/recovered.edits/0000000000000000026 2024-11-12T10:30:31,049 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:31,051 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:31,062 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829/wal.1731407431051, exclude list is [], retry=0 2024-11-12T10:30:31,064 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:31,064 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:31,064 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:31,066 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829/wal.1731407431051 2024-11-12T10:30:31,066 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33367:33367)] 2024-11-12T10:30:31,066 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 97a2e4b249b2d44cb1d98a1ca3ee3ae1, NAME => 'testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:31,067 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:31,067 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:31,067 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:31,067 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:31,069 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:31,070 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97a2e4b249b2d44cb1d98a1ca3ee3ae1 columnFamilyName a 2024-11-12T10:30:31,070 DEBUG [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:31,071 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] regionserver.HStore(327): Store=97a2e4b249b2d44cb1d98a1ca3ee3ae1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:31,071 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:31,071 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97a2e4b249b2d44cb1d98a1ca3ee3ae1 columnFamilyName b 2024-11-12T10:30:31,071 DEBUG [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:31,072 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] regionserver.HStore(327): Store=97a2e4b249b2d44cb1d98a1ca3ee3ae1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:31,072 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:31,073 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97a2e4b249b2d44cb1d98a1ca3ee3ae1 columnFamilyName c 2024-11-12T10:30:31,073 DEBUG [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:31,073 INFO [StoreOpener-97a2e4b249b2d44cb1d98a1ca3ee3ae1-1 {}] regionserver.HStore(327): Store=97a2e4b249b2d44cb1d98a1ca3ee3ae1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:31,073 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:31,074 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:31,075 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:31,076 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/recovered.edits/0000000000000000026 2024-11-12T10:30:31,078 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/recovered.edits/0000000000000000026 2024-11-12T10:30:31,079 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 97a2e4b249b2d44cb1d98a1ca3ee3ae1 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-12T10:30:31,093 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/.tmp/a/6e0d75f4ac7144e3869d3613b9749666 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731407431005/Put/seqid=0 2024-11-12T10:30:31,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741911_1089 (size=5523) 2024-11-12T10:30:31,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741911_1089 (size=5523) 2024-11-12T10:30:31,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741911_1089 (size=5523) 2024-11-12T10:30:31,100 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/.tmp/a/6e0d75f4ac7144e3869d3613b9749666 2024-11-12T10:30:31,117 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/.tmp/b/306ce1c64d3242b0b886721d2fc231af is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731407431000/Put/seqid=0 2024-11-12T10:30:31,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741912_1090 (size=5524) 2024-11-12T10:30:31,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741912_1090 (size=5524) 2024-11-12T10:30:31,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741912_1090 (size=5524) 2024-11-12T10:30:31,125 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/.tmp/b/306ce1c64d3242b0b886721d2fc231af 2024-11-12T10:30:31,144 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/.tmp/c/5b5d0f3ce6294861bd61e8f247eef97c is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731407431002/Put/seqid=0 2024-11-12T10:30:31,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741913_1091 (size=5457) 2024-11-12T10:30:31,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741913_1091 (size=5457) 2024-11-12T10:30:31,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741913_1091 (size=5457) 2024-11-12T10:30:31,152 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/.tmp/c/5b5d0f3ce6294861bd61e8f247eef97c 2024-11-12T10:30:31,158 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/.tmp/a/6e0d75f4ac7144e3869d3613b9749666 as hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/a/6e0d75f4ac7144e3869d3613b9749666 2024-11-12T10:30:31,165 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/a/6e0d75f4ac7144e3869d3613b9749666, entries=7, sequenceid=26, filesize=5.4 K 2024-11-12T10:30:31,166 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/.tmp/b/306ce1c64d3242b0b886721d2fc231af as hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/b/306ce1c64d3242b0b886721d2fc231af 2024-11-12T10:30:31,172 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/b/306ce1c64d3242b0b886721d2fc231af, entries=7, sequenceid=26, filesize=5.4 K 2024-11-12T10:30:31,173 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/.tmp/c/5b5d0f3ce6294861bd61e8f247eef97c as hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/c/5b5d0f3ce6294861bd61e8f247eef97c 2024-11-12T10:30:31,178 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/c/5b5d0f3ce6294861bd61e8f247eef97c, entries=6, sequenceid=26, filesize=5.3 K 2024-11-12T10:30:31,179 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 in 100ms, sequenceid=26, compaction requested=false; wal=null 2024-11-12T10:30:31,179 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/recovered.edits/0000000000000000026 2024-11-12T10:30:31,181 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:31,181 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:31,181 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T10:30:31,183 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:31,185 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testReplayEditsAfterAbortingFlush/97a2e4b249b2d44cb1d98a1ca3ee3ae1/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-12T10:30:31,186 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 97a2e4b249b2d44cb1d98a1ca3ee3ae1; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58761211, jitterRate=-0.12438972294330597}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T10:30:31,187 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 2024-11-12T10:30:31,187 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 97a2e4b249b2d44cb1d98a1ca3ee3ae1: Running coprocessor pre-open hook at 1731407431067Writing region info on filesystem at 1731407431067Initializing all the Stores at 1731407431068 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407431068Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407431069 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407431069Obtaining lock to block concurrent updates at 1731407431079 (+10 ms)Preparing flush snapshotting stores in 97a2e4b249b2d44cb1d98a1ca3ee3ae1 at 1731407431079Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731407431079Flushing stores of testReplayEditsAfterAbortingFlush,,1731407421831.97a2e4b249b2d44cb1d98a1ca3ee3ae1. at 1731407431079Flushing 97a2e4b249b2d44cb1d98a1ca3ee3ae1/a: creating writer at 1731407431079Flushing 97a2e4b249b2d44cb1d98a1ca3ee3ae1/a: appending metadata at 1731407431093 (+14 ms)Flushing 97a2e4b249b2d44cb1d98a1ca3ee3ae1/a: closing flushed file at 1731407431093Flushing 97a2e4b249b2d44cb1d98a1ca3ee3ae1/b: creating writer at 1731407431105 (+12 ms)Flushing 97a2e4b249b2d44cb1d98a1ca3ee3ae1/b: appending metadata at 1731407431117 (+12 ms)Flushing 97a2e4b249b2d44cb1d98a1ca3ee3ae1/b: closing flushed file at 1731407431117Flushing 97a2e4b249b2d44cb1d98a1ca3ee3ae1/c: creating writer at 1731407431130 (+13 ms)Flushing 97a2e4b249b2d44cb1d98a1ca3ee3ae1/c: appending metadata at 1731407431144 (+14 ms)Flushing 97a2e4b249b2d44cb1d98a1ca3ee3ae1/c: closing flushed file at 1731407431144Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23a01856: reopening flushed file at 1731407431157 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e8c5583: reopening flushed file at 1731407431165 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f858d8f: reopening flushed file at 1731407431172 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 97a2e4b249b2d44cb1d98a1ca3ee3ae1 in 100ms, sequenceid=26, compaction requested=false; wal=null at 1731407431179 (+7 ms)Cleaning up temporary data from old regions at 1731407431181 (+2 ms)Running coprocessor post-open hooks at 1731407431187 (+6 ms)Region opened successfully at 1731407431187 2024-11-12T10:30:31,208 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415 (was 413) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:49730 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:36488 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:37576 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:37556 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-663294313-172.17.0.3-1731407397077:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-663294313-172.17.0.3-1731407397077:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:36510 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:49712 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-663294313-172.17.0.3-1731407397077:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1178 (was 1118) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=572 (was 676), ProcessCount=11 (was 11), AvailableMemoryMB=7233 (was 7254) 2024-11-12T10:30:31,208 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1178 is superior to 1024 2024-11-12T10:30:31,219 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=415, OpenFileDescriptor=1178, MaxFileDescriptor=1048576, SystemLoadAverage=572, ProcessCount=11, AvailableMemoryMB=7232 2024-11-12T10:30:31,220 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1178 is superior to 1024 2024-11-12T10:30:31,233 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:31,235 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:31,235 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T10:30:31,238 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-29442096, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/hregion-29442096, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:31,250 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-29442096/hregion-29442096.1731407431238, exclude list is [], retry=0 2024-11-12T10:30:31,253 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:31,253 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:31,254 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:31,255 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-29442096/hregion-29442096.1731407431238 2024-11-12T10:30:31,255 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33413:33413)] 2024-11-12T10:30:31,256 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1557b5b953fbfea991d2b693b10f64b5, NAME => 'testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38657/hbase 2024-11-12T10:30:31,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741915_1093 (size=61) 2024-11-12T10:30:31,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741915_1093 (size=61) 2024-11-12T10:30:31,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741915_1093 (size=61) 2024-11-12T10:30:31,264 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:31,265 INFO [StoreOpener-1557b5b953fbfea991d2b693b10f64b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,267 INFO [StoreOpener-1557b5b953fbfea991d2b693b10f64b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1557b5b953fbfea991d2b693b10f64b5 columnFamilyName a 2024-11-12T10:30:31,267 DEBUG [StoreOpener-1557b5b953fbfea991d2b693b10f64b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:31,268 INFO [StoreOpener-1557b5b953fbfea991d2b693b10f64b5-1 {}] regionserver.HStore(327): Store=1557b5b953fbfea991d2b693b10f64b5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:31,268 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,269 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,269 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,270 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,270 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,271 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,274 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:30:31,274 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1557b5b953fbfea991d2b693b10f64b5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58941677, jitterRate=-0.12170056998729706}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T10:30:31,275 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1557b5b953fbfea991d2b693b10f64b5: Writing region info on filesystem at 1731407431264Initializing all the Stores at 1731407431265 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407431265Cleaning up temporary data from old regions at 1731407431270 (+5 ms)Region opened successfully at 1731407431275 (+5 ms) 2024-11-12T10:30:31,275 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1557b5b953fbfea991d2b693b10f64b5, disabling compactions & flushes 2024-11-12T10:30:31,275 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5. 2024-11-12T10:30:31,275 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5. 2024-11-12T10:30:31,275 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5. after waiting 0 ms 2024-11-12T10:30:31,275 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5. 2024-11-12T10:30:31,276 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5. 2024-11-12T10:30:31,276 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1557b5b953fbfea991d2b693b10f64b5: Waiting for close lock at 1731407431275Disabling compacts and flushes for region at 1731407431275Disabling writes for close at 1731407431275Writing region close event to WAL at 1731407431276 (+1 ms)Closed at 1731407431276 2024-11-12T10:30:31,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741914_1092 (size=93) 2024-11-12T10:30:31,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741914_1092 (size=93) 2024-11-12T10:30:31,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741914_1092 (size=93) 2024-11-12T10:30:31,281 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T10:30:31,281 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-29442096:(num 1731407431238) 2024-11-12T10:30:31,281 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:31,283 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:31,295 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232/wal.1731407431283, exclude list is [], retry=0 2024-11-12T10:30:31,298 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:31,298 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:31,298 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:31,300 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232/wal.1731407431283 2024-11-12T10:30:31,300 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:33413:33413)] 2024-11-12T10:30:31,300 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1557b5b953fbfea991d2b693b10f64b5, NAME => 'testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:31,300 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:31,300 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,300 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,302 INFO [StoreOpener-1557b5b953fbfea991d2b693b10f64b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,303 INFO [StoreOpener-1557b5b953fbfea991d2b693b10f64b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1557b5b953fbfea991d2b693b10f64b5 columnFamilyName a 2024-11-12T10:30:31,303 DEBUG [StoreOpener-1557b5b953fbfea991d2b693b10f64b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:31,304 INFO [StoreOpener-1557b5b953fbfea991d2b693b10f64b5-1 {}] regionserver.HStore(327): Store=1557b5b953fbfea991d2b693b10f64b5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:31,304 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,304 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,306 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,306 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,306 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,308 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,309 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1557b5b953fbfea991d2b693b10f64b5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75488966, jitterRate=0.12487325072288513}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T10:30:31,309 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1557b5b953fbfea991d2b693b10f64b5: Writing region info on filesystem at 1731407431301Initializing all the Stores at 1731407431301Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407431301Cleaning up temporary data from old regions at 1731407431306 (+5 ms)Region opened successfully at 1731407431309 (+3 ms) 2024-11-12T10:30:31,317 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1557b5b953fbfea991d2b693b10f64b5, disabling compactions & flushes 2024-11-12T10:30:31,317 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5. 2024-11-12T10:30:31,317 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5. 2024-11-12T10:30:31,317 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5. after waiting 0 ms 2024-11-12T10:30:31,317 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5. 2024-11-12T10:30:31,318 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5. 2024-11-12T10:30:31,318 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5. 2024-11-12T10:30:31,318 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1557b5b953fbfea991d2b693b10f64b5: Waiting for close lock at 1731407431317Disabling compacts and flushes for region at 1731407431317Disabling writes for close at 1731407431317Writing region close event to WAL at 1731407431318 (+1 ms)Closed at 1731407431318 2024-11-12T10:30:31,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741916_1094 (size=1623) 2024-11-12T10:30:31,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741916_1094 (size=1623) 2024-11-12T10:30:31,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741916_1094 (size=1623) 2024-11-12T10:30:31,342 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38657/hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232/wal.1731407431283, size=1.6 K (1623bytes) 2024-11-12T10:30:31,342 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38657/hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232/wal.1731407431283 2024-11-12T10:30:31,343 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38657/hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232/wal.1731407431283 after 1ms 2024-11-12T10:30:31,346 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38657/hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232/wal.1731407431283 took 4ms 2024-11-12T10:30:31,348 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38657/hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232/wal.1731407431283 so closing down 2024-11-12T10:30:31,348 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T10:30:31,349 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731407431283.temp 2024-11-12T10:30:31,350 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/recovered.edits/0000000000000000003-wal.1731407431283.temp 2024-11-12T10:30:31,351 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T10:30:31,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741917_1095 (size=1623) 2024-11-12T10:30:31,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741917_1095 (size=1623) 2024-11-12T10:30:31,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741917_1095 (size=1623) 2024-11-12T10:30:31,358 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/recovered.edits/0000000000000000003-wal.1731407431283.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-12T10:30:31,359 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/recovered.edits/0000000000000000003-wal.1731407431283.temp to hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/recovered.edits/0000000000000000012 2024-11-12T10:30:31,359 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:38657/hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232/wal.1731407431283, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-11-12T10:30:31,359 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38657/hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232/wal.1731407431283, journal: Splitting hdfs://localhost:38657/hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232/wal.1731407431283, size=1.6 K (1623bytes) at 1731407431342Finishing writing output for hdfs://localhost:38657/hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232/wal.1731407431283 so closing down at 1731407431348 (+6 ms)Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/recovered.edits/0000000000000000003-wal.1731407431283.temp at 1731407431350 (+2 ms)3 split writer threads finished at 1731407431351 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/recovered.edits/0000000000000000003-wal.1731407431283.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731407431358 (+7 ms)Rename recovered edits hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/recovered.edits/0000000000000000003-wal.1731407431283.temp to hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/recovered.edits/0000000000000000012 at 1731407431359 (+1 ms)Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:38657/hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232/wal.1731407431283, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1731407431359 2024-11-12T10:30:31,361 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38657/hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232/wal.1731407431283 to hdfs://localhost:38657/hbase/oldWALs/wal.1731407431283 2024-11-12T10:30:31,362 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/recovered.edits/0000000000000000012 2024-11-12T10:30:31,710 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:31,712 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:31,723 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232/wal.1731407431712, exclude list is [], retry=0 2024-11-12T10:30:31,725 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:31,726 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:31,726 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:31,727 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232/wal.1731407431712 2024-11-12T10:30:31,728 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33413:33413)] 2024-11-12T10:30:31,728 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1557b5b953fbfea991d2b693b10f64b5, NAME => 'testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:31,728 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:31,728 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,728 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,732 INFO [StoreOpener-1557b5b953fbfea991d2b693b10f64b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,734 INFO [StoreOpener-1557b5b953fbfea991d2b693b10f64b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1557b5b953fbfea991d2b693b10f64b5 columnFamilyName a 2024-11-12T10:30:31,734 DEBUG [StoreOpener-1557b5b953fbfea991d2b693b10f64b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:31,735 INFO [StoreOpener-1557b5b953fbfea991d2b693b10f64b5-1 {}] regionserver.HStore(327): Store=1557b5b953fbfea991d2b693b10f64b5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:31,735 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,736 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,737 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,737 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/recovered.edits/0000000000000000012 2024-11-12T10:30:31,740 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/recovered.edits/0000000000000000012 2024-11-12T10:30:31,740 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1557b5b953fbfea991d2b693b10f64b5 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-12T10:30:31,756 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/.tmp/a/3b37b5c1e17e4c19a7d30b58322e89fd is 79, key is testDatalossWhenInputError/a:x0/1731407431309/Put/seqid=0 2024-11-12T10:30:31,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741919_1097 (size=5808) 2024-11-12T10:30:31,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741919_1097 (size=5808) 2024-11-12T10:30:31,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741919_1097 (size=5808) 2024-11-12T10:30:31,764 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/.tmp/a/3b37b5c1e17e4c19a7d30b58322e89fd 2024-11-12T10:30:31,773 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/.tmp/a/3b37b5c1e17e4c19a7d30b58322e89fd as hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/a/3b37b5c1e17e4c19a7d30b58322e89fd 2024-11-12T10:30:31,782 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/a/3b37b5c1e17e4c19a7d30b58322e89fd, entries=10, sequenceid=12, filesize=5.7 K 2024-11-12T10:30:31,782 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 1557b5b953fbfea991d2b693b10f64b5 in 42ms, sequenceid=12, compaction requested=false; wal=null 2024-11-12T10:30:31,783 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/recovered.edits/0000000000000000012 2024-11-12T10:30:31,784 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,784 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,787 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,790 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-12T10:30:31,791 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1557b5b953fbfea991d2b693b10f64b5; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61169683, jitterRate=-0.08850069344043732}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T10:30:31,791 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1557b5b953fbfea991d2b693b10f64b5: Writing region info on filesystem at 1731407431728Initializing all the Stores at 1731407431731 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407431731Obtaining lock to block concurrent updates at 1731407431740 (+9 ms)Preparing flush snapshotting stores in 1557b5b953fbfea991d2b693b10f64b5 at 1731407431740Finished memstore snapshotting testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731407431740Flushing stores of testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5. at 1731407431740Flushing 1557b5b953fbfea991d2b693b10f64b5/a: creating writer at 1731407431740Flushing 1557b5b953fbfea991d2b693b10f64b5/a: appending metadata at 1731407431756 (+16 ms)Flushing 1557b5b953fbfea991d2b693b10f64b5/a: closing flushed file at 1731407431756Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21aa7814: reopening flushed file at 1731407431771 (+15 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 1557b5b953fbfea991d2b693b10f64b5 in 42ms, sequenceid=12, compaction requested=false; wal=null at 1731407431782 (+11 ms)Cleaning up temporary data from old regions at 1731407431784 (+2 ms)Region opened successfully at 1731407431791 (+7 ms) 2024-11-12T10:30:31,794 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1557b5b953fbfea991d2b693b10f64b5, NAME => 'testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:31,794 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731407431233.1557b5b953fbfea991d2b693b10f64b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:31,794 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,795 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,796 INFO [StoreOpener-1557b5b953fbfea991d2b693b10f64b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,797 INFO [StoreOpener-1557b5b953fbfea991d2b693b10f64b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1557b5b953fbfea991d2b693b10f64b5 columnFamilyName a 2024-11-12T10:30:31,797 DEBUG [StoreOpener-1557b5b953fbfea991d2b693b10f64b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:31,802 DEBUG [StoreOpener-1557b5b953fbfea991d2b693b10f64b5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/a/3b37b5c1e17e4c19a7d30b58322e89fd 2024-11-12T10:30:31,802 INFO [StoreOpener-1557b5b953fbfea991d2b693b10f64b5-1 {}] regionserver.HStore(327): Store=1557b5b953fbfea991d2b693b10f64b5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:31,802 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,803 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,804 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,804 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,804 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,806 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1557b5b953fbfea991d2b693b10f64b5 2024-11-12T10:30:31,808 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testDatalossWhenInputError/1557b5b953fbfea991d2b693b10f64b5/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-12T10:30:31,809 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1557b5b953fbfea991d2b693b10f64b5; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63501510, jitterRate=-0.05375376343727112}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T10:30:31,809 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1557b5b953fbfea991d2b693b10f64b5: Writing region info on filesystem at 1731407431795Initializing all the Stores at 1731407431795Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407431796 (+1 ms)Cleaning up temporary data from old regions at 1731407431804 (+8 ms)Region opened successfully at 1731407431809 (+5 ms) 2024-11-12T10:30:31,826 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=425 (was 415) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:49730 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:37660 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:36558 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:49784 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-663294313-172.17.0.3-1731407397077:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-663294313-172.17.0.3-1731407397077:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:37556 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-663294313-172.17.0.3-1731407397077:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1260 (was 1178) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=572 (was 572), ProcessCount=11 (was 11), AvailableMemoryMB=7219 (was 7232) 2024-11-12T10:30:31,826 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1260 is superior to 1024 2024-11-12T10:30:31,837 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=425, OpenFileDescriptor=1260, MaxFileDescriptor=1048576, SystemLoadAverage=572, ProcessCount=11, AvailableMemoryMB=7218 2024-11-12T10:30:31,838 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1260 is superior to 1024 2024-11-12T10:30:31,852 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:31,854 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:31,854 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T10:30:31,857 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-92966089, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/hregion-92966089, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:31,869 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-92966089/hregion-92966089.1731407431857, exclude list is [], retry=0 2024-11-12T10:30:31,872 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:31,873 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:31,873 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:31,880 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-92966089/hregion-92966089.1731407431857 2024-11-12T10:30:31,881 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:39823:39823)] 2024-11-12T10:30:31,881 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2da37b5e3a18d9194bcfa5eb39a145a4, NAME => 'testCompactedBulkLoadedFiles,,1731407431852.2da37b5e3a18d9194bcfa5eb39a145a4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38657/hbase 2024-11-12T10:30:31,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741921_1099 (size=63) 2024-11-12T10:30:31,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741921_1099 (size=63) 2024-11-12T10:30:31,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741921_1099 (size=63) 2024-11-12T10:30:31,893 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731407431852.2da37b5e3a18d9194bcfa5eb39a145a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:31,894 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,895 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2da37b5e3a18d9194bcfa5eb39a145a4 columnFamilyName a 2024-11-12T10:30:31,895 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:31,896 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.HStore(327): Store=2da37b5e3a18d9194bcfa5eb39a145a4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:31,896 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,897 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2da37b5e3a18d9194bcfa5eb39a145a4 columnFamilyName b 2024-11-12T10:30:31,897 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:31,898 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.HStore(327): Store=2da37b5e3a18d9194bcfa5eb39a145a4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:31,898 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,900 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2da37b5e3a18d9194bcfa5eb39a145a4 columnFamilyName c 2024-11-12T10:30:31,900 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:31,900 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.HStore(327): Store=2da37b5e3a18d9194bcfa5eb39a145a4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:31,900 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,901 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,902 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,903 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,903 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,903 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T10:30:31,905 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,907 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:30:31,907 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2da37b5e3a18d9194bcfa5eb39a145a4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64682663, jitterRate=-0.03615321218967438}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T10:30:31,908 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2da37b5e3a18d9194bcfa5eb39a145a4: Writing region info on filesystem at 1731407431893Initializing all the Stores at 1731407431893Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407431894 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407431894Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407431894Cleaning up temporary data from old regions at 1731407431903 (+9 ms)Region opened successfully at 1731407431908 (+5 ms) 2024-11-12T10:30:31,908 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2da37b5e3a18d9194bcfa5eb39a145a4, disabling compactions & flushes 2024-11-12T10:30:31,908 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731407431852.2da37b5e3a18d9194bcfa5eb39a145a4. 2024-11-12T10:30:31,908 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731407431852.2da37b5e3a18d9194bcfa5eb39a145a4. 2024-11-12T10:30:31,908 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731407431852.2da37b5e3a18d9194bcfa5eb39a145a4. after waiting 0 ms 2024-11-12T10:30:31,908 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731407431852.2da37b5e3a18d9194bcfa5eb39a145a4. 2024-11-12T10:30:31,909 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731407431852.2da37b5e3a18d9194bcfa5eb39a145a4. 2024-11-12T10:30:31,909 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2da37b5e3a18d9194bcfa5eb39a145a4: Waiting for close lock at 1731407431908Disabling compacts and flushes for region at 1731407431908Disabling writes for close at 1731407431908Writing region close event to WAL at 1731407431909 (+1 ms)Closed at 1731407431909 2024-11-12T10:30:31,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741920_1098 (size=93) 2024-11-12T10:30:31,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741920_1098 (size=93) 2024-11-12T10:30:31,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741920_1098 (size=93) 2024-11-12T10:30:31,914 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T10:30:31,914 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-92966089:(num 1731407431857) 2024-11-12T10:30:31,914 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:31,916 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:31,928 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407431916, exclude list is [], retry=0 2024-11-12T10:30:31,931 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:31,931 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:31,931 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:31,933 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407431916 2024-11-12T10:30:31,935 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:33413:33413)] 2024-11-12T10:30:31,935 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2da37b5e3a18d9194bcfa5eb39a145a4, NAME => 'testCompactedBulkLoadedFiles,,1731407431852.2da37b5e3a18d9194bcfa5eb39a145a4.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:31,935 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731407431852.2da37b5e3a18d9194bcfa5eb39a145a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:31,936 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,936 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,937 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,938 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2da37b5e3a18d9194bcfa5eb39a145a4 columnFamilyName a 2024-11-12T10:30:31,938 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:31,939 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.HStore(327): Store=2da37b5e3a18d9194bcfa5eb39a145a4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:31,939 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,940 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2da37b5e3a18d9194bcfa5eb39a145a4 columnFamilyName b 2024-11-12T10:30:31,940 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:31,940 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.HStore(327): Store=2da37b5e3a18d9194bcfa5eb39a145a4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:31,940 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,941 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2da37b5e3a18d9194bcfa5eb39a145a4 columnFamilyName c 2024-11-12T10:30:31,941 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:31,942 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.HStore(327): Store=2da37b5e3a18d9194bcfa5eb39a145a4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:31,942 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,942 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,944 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,945 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,946 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,946 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T10:30:31,947 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:31,948 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2da37b5e3a18d9194bcfa5eb39a145a4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75311483, jitterRate=0.12222854793071747}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T10:30:31,949 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2da37b5e3a18d9194bcfa5eb39a145a4: Writing region info on filesystem at 1731407431936Initializing all the Stores at 1731407431937 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407431937Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407431937Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407431937Cleaning up temporary data from old regions at 1731407431946 (+9 ms)Region opened successfully at 1731407431949 (+3 ms) 2024-11-12T10:30:31,953 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731407431953/Put/seqid=0 2024-11-12T10:30:31,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741923_1101 (size=4875) 2024-11-12T10:30:31,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741923_1101 (size=4875) 2024-11-12T10:30:31,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741923_1101 (size=4875) 2024-11-12T10:30:31,965 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731407431964/Put/seqid=0 2024-11-12T10:30:31,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741924_1102 (size=4875) 2024-11-12T10:30:31,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741924_1102 (size=4875) 2024-11-12T10:30:31,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741924_1102 (size=4875) 2024-11-12T10:30:31,977 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731407431976/Put/seqid=0 2024-11-12T10:30:31,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741925_1103 (size=4875) 2024-11-12T10:30:31,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741925_1103 (size=4875) 2024-11-12T10:30:31,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741925_1103 (size=4875) 2024-11-12T10:30:31,984 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38657/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 2da37b5e3a18d9194bcfa5eb39a145a4/a 2024-11-12T10:30:31,987 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-12T10:30:31,987 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-12T10:30:31,987 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38657/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 2da37b5e3a18d9194bcfa5eb39a145a4/a 2024-11-12T10:30:31,991 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-12T10:30:31,991 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-12T10:30:31,991 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38657/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 2da37b5e3a18d9194bcfa5eb39a145a4/a 2024-11-12T10:30:31,994 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-12T10:30:31,994 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-12T10:30:31,994 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2da37b5e3a18d9194bcfa5eb39a145a4 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-12T10:30:32,005 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-12T10:30:32,005 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-12T10:30:32,006 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-12T10:30:32,006 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-12T10:30:32,009 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/.tmp/a/58e4e4c639dc40dc99a16d56471cfaf9 is 55, key is testCompactedBulkLoadedFiles/a:a/1731407431950/Put/seqid=0 2024-11-12T10:30:32,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741926_1104 (size=5107) 2024-11-12T10:30:32,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741926_1104 (size=5107) 2024-11-12T10:30:32,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741926_1104 (size=5107) 2024-11-12T10:30:32,015 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/.tmp/a/58e4e4c639dc40dc99a16d56471cfaf9 2024-11-12T10:30:32,020 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/.tmp/a/58e4e4c639dc40dc99a16d56471cfaf9 as hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/58e4e4c639dc40dc99a16d56471cfaf9 2024-11-12T10:30:32,026 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/58e4e4c639dc40dc99a16d56471cfaf9, entries=1, sequenceid=4, filesize=5.0 K 2024-11-12T10:30:32,027 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 2da37b5e3a18d9194bcfa5eb39a145a4 in 33ms, sequenceid=4, compaction requested=false 2024-11-12T10:30:32,027 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2da37b5e3a18d9194bcfa5eb39a145a4: 2024-11-12T10:30:32,029 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/12d7ec5e5905457e9f473f187af4a2ff_SeqId_4_ 2024-11-12T10:30:32,030 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/2879cc1df97740fba10fec5af3086cde_SeqId_4_ 2024-11-12T10:30:32,031 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/18b9a6d2c15846ec83a892d52c2ac5ab_SeqId_4_ 2024-11-12T10:30:32,032 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38657/hbase/testCompactedBulkLoadedFiles/hfile0 into 2da37b5e3a18d9194bcfa5eb39a145a4/a as hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/12d7ec5e5905457e9f473f187af4a2ff_SeqId_4_ - updating store file list. 2024-11-12T10:30:32,036 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 12d7ec5e5905457e9f473f187af4a2ff_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-12T10:30:32,036 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/12d7ec5e5905457e9f473f187af4a2ff_SeqId_4_ into 2da37b5e3a18d9194bcfa5eb39a145a4/a 2024-11-12T10:30:32,036 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38657/hbase/testCompactedBulkLoadedFiles/hfile0 into 2da37b5e3a18d9194bcfa5eb39a145a4/a (new location: hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/12d7ec5e5905457e9f473f187af4a2ff_SeqId_4_) 2024-11-12T10:30:32,037 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38657/hbase/testCompactedBulkLoadedFiles/hfile1 into 2da37b5e3a18d9194bcfa5eb39a145a4/a as hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/2879cc1df97740fba10fec5af3086cde_SeqId_4_ - updating store file list. 2024-11-12T10:30:32,041 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2879cc1df97740fba10fec5af3086cde_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-12T10:30:32,041 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/2879cc1df97740fba10fec5af3086cde_SeqId_4_ into 2da37b5e3a18d9194bcfa5eb39a145a4/a 2024-11-12T10:30:32,041 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38657/hbase/testCompactedBulkLoadedFiles/hfile1 into 2da37b5e3a18d9194bcfa5eb39a145a4/a (new location: hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/2879cc1df97740fba10fec5af3086cde_SeqId_4_) 2024-11-12T10:30:32,042 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38657/hbase/testCompactedBulkLoadedFiles/hfile2 into 2da37b5e3a18d9194bcfa5eb39a145a4/a as hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/18b9a6d2c15846ec83a892d52c2ac5ab_SeqId_4_ - updating store file list. 2024-11-12T10:30:32,046 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 18b9a6d2c15846ec83a892d52c2ac5ab_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-12T10:30:32,046 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/18b9a6d2c15846ec83a892d52c2ac5ab_SeqId_4_ into 2da37b5e3a18d9194bcfa5eb39a145a4/a 2024-11-12T10:30:32,046 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38657/hbase/testCompactedBulkLoadedFiles/hfile2 into 2da37b5e3a18d9194bcfa5eb39a145a4/a (new location: hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/18b9a6d2c15846ec83a892d52c2ac5ab_SeqId_4_) 2024-11-12T10:30:32,053 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T10:30:32,053 DEBUG [Time-limited test {}] regionserver.HStore(1541): 2da37b5e3a18d9194bcfa5eb39a145a4/a is initiating major compaction (all files) 2024-11-12T10:30:32,053 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 2da37b5e3a18d9194bcfa5eb39a145a4/a in testCompactedBulkLoadedFiles,,1731407431852.2da37b5e3a18d9194bcfa5eb39a145a4. 2024-11-12T10:30:32,054 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/58e4e4c639dc40dc99a16d56471cfaf9, hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/12d7ec5e5905457e9f473f187af4a2ff_SeqId_4_, hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/2879cc1df97740fba10fec5af3086cde_SeqId_4_, hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/18b9a6d2c15846ec83a892d52c2ac5ab_SeqId_4_] into tmpdir=hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/.tmp, totalSize=19.3 K 2024-11-12T10:30:32,054 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 58e4e4c639dc40dc99a16d56471cfaf9, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731407431950 2024-11-12T10:30:32,055 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 12d7ec5e5905457e9f473f187af4a2ff_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-12T10:30:32,055 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2879cc1df97740fba10fec5af3086cde_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-12T10:30:32,055 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 18b9a6d2c15846ec83a892d52c2ac5ab_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-12T10:30:32,069 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/.tmp/a/39e21b8571904fd783b21bdc4764e24a is 55, key is testCompactedBulkLoadedFiles/a:a/1731407431950/Put/seqid=0 2024-11-12T10:30:32,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741927_1105 (size=6154) 2024-11-12T10:30:32,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741927_1105 (size=6154) 2024-11-12T10:30:32,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741927_1105 (size=6154) 2024-11-12T10:30:32,081 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/.tmp/a/39e21b8571904fd783b21bdc4764e24a as hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/39e21b8571904fd783b21bdc4764e24a 2024-11-12T10:30:32,089 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 2da37b5e3a18d9194bcfa5eb39a145a4/a of 2da37b5e3a18d9194bcfa5eb39a145a4 into 39e21b8571904fd783b21bdc4764e24a(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T10:30:32,089 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 2da37b5e3a18d9194bcfa5eb39a145a4: 2024-11-12T10:30:32,089 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-12T10:30:32,089 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-12T10:30:32,121 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407431916, size=0 (0bytes) 2024-11-12T10:30:32,121 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407431916 might be still open, length is 0 2024-11-12T10:30:32,121 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407431916 2024-11-12T10:30:32,122 WARN [IPC Server handler 3 on default port 38657 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407431916 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-12T10:30:32,122 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407431916 after 1ms 2024-11-12T10:30:33,720 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:46744 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:43905:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46744 dst: /127.0.0.1:43905 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43905 remote=/127.0.0.1:46744]. Total timeout mills is 60000, 58369 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:33,721 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:35462 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:34283:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35462 dst: /127.0.0.1:34283 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:33,721 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:43678 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:35741:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43678 dst: /127.0.0.1:35741 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:33,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741922_1106 (size=1659) 2024-11-12T10:30:33,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741922_1106 (size=1659) 2024-11-12T10:30:33,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741922_1106 (size=1659) 2024-11-12T10:30:36,124 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407431916 after 4003ms 2024-11-12T10:30:36,130 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407431916 took 4008ms 2024-11-12T10:30:36,132 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407431916; continuing. 2024-11-12T10:30:36,132 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407431916 so closing down 2024-11-12T10:30:36,132 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T10:30:36,134 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731407431916.temp 2024-11-12T10:30:36,136 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/recovered.edits/0000000000000000003-wal.1731407431916.temp 2024-11-12T10:30:36,136 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T10:30:36,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741928_1107 (size=698) 2024-11-12T10:30:36,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741928_1107 (size=698) 2024-11-12T10:30:36,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741928_1107 (size=698) 2024-11-12T10:30:36,145 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/recovered.edits/0000000000000000003-wal.1731407431916.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-12T10:30:36,147 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/recovered.edits/0000000000000000003-wal.1731407431916.temp to hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/recovered.edits/0000000000000000008 2024-11-12T10:30:36,147 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:38657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407431916, size=0, length=0, corrupted=false, cancelled=false 2024-11-12T10:30:36,147 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407431916, journal: Splitting hdfs://localhost:38657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407431916, size=0 (0bytes) at 1731407432121Finishing writing output for hdfs://localhost:38657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407431916 so closing down at 1731407436132 (+4011 ms)Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/recovered.edits/0000000000000000003-wal.1731407431916.temp at 1731407436136 (+4 ms)3 split writer threads finished at 1731407436136Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/recovered.edits/0000000000000000003-wal.1731407431916.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731407436146 (+10 ms)Rename recovered edits hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/recovered.edits/0000000000000000003-wal.1731407431916.temp to hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/recovered.edits/0000000000000000008 at 1731407436147 (+1 ms)Processed 5 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:38657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407431916, size=0, length=0, corrupted=false, cancelled=false at 1731407436147 2024-11-12T10:30:36,148 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407431916 to hdfs://localhost:38657/hbase/oldWALs/wal.1731407431916 2024-11-12T10:30:36,149 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/recovered.edits/0000000000000000008 2024-11-12T10:30:36,149 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:36,151 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:36,162 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407436151, exclude list is [], retry=0 2024-11-12T10:30:36,165 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:36,165 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:36,165 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:36,166 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407436151 2024-11-12T10:30:36,167 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:39823:39823)] 2024-11-12T10:30:36,167 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2da37b5e3a18d9194bcfa5eb39a145a4, NAME => 'testCompactedBulkLoadedFiles,,1731407431852.2da37b5e3a18d9194bcfa5eb39a145a4.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:36,167 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731407431852.2da37b5e3a18d9194bcfa5eb39a145a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:36,167 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:36,167 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:36,168 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:36,169 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2da37b5e3a18d9194bcfa5eb39a145a4 columnFamilyName a 2024-11-12T10:30:36,169 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:36,174 DEBUG [StoreFileOpener-2da37b5e3a18d9194bcfa5eb39a145a4-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 12d7ec5e5905457e9f473f187af4a2ff_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-12T10:30:36,175 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/12d7ec5e5905457e9f473f187af4a2ff_SeqId_4_ 2024-11-12T10:30:36,177 DEBUG [StoreFileOpener-2da37b5e3a18d9194bcfa5eb39a145a4-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 18b9a6d2c15846ec83a892d52c2ac5ab_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-12T10:30:36,177 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/18b9a6d2c15846ec83a892d52c2ac5ab_SeqId_4_ 2024-11-12T10:30:36,180 DEBUG [StoreFileOpener-2da37b5e3a18d9194bcfa5eb39a145a4-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2879cc1df97740fba10fec5af3086cde_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-12T10:30:36,180 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/2879cc1df97740fba10fec5af3086cde_SeqId_4_ 2024-11-12T10:30:36,183 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/39e21b8571904fd783b21bdc4764e24a 2024-11-12T10:30:36,187 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/58e4e4c639dc40dc99a16d56471cfaf9 2024-11-12T10:30:36,187 WARN [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/12d7ec5e5905457e9f473f187af4a2ff_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1c5825c5 2024-11-12T10:30:36,187 WARN [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/18b9a6d2c15846ec83a892d52c2ac5ab_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1c5825c5 2024-11-12T10:30:36,188 WARN [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/2879cc1df97740fba10fec5af3086cde_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1c5825c5 2024-11-12T10:30:36,188 WARN [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/58e4e4c639dc40dc99a16d56471cfaf9 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1c5825c5 2024-11-12T10:30:36,188 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/12d7ec5e5905457e9f473f187af4a2ff_SeqId_4_, hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/18b9a6d2c15846ec83a892d52c2ac5ab_SeqId_4_, hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/2879cc1df97740fba10fec5af3086cde_SeqId_4_, hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/58e4e4c639dc40dc99a16d56471cfaf9] to archive 2024-11-12T10:30:36,189 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T10:30:36,190 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/12d7ec5e5905457e9f473f187af4a2ff_SeqId_4_ to hdfs://localhost:38657/hbase/archive/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/12d7ec5e5905457e9f473f187af4a2ff_SeqId_4_ 2024-11-12T10:30:36,191 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/18b9a6d2c15846ec83a892d52c2ac5ab_SeqId_4_ to hdfs://localhost:38657/hbase/archive/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/18b9a6d2c15846ec83a892d52c2ac5ab_SeqId_4_ 2024-11-12T10:30:36,192 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/2879cc1df97740fba10fec5af3086cde_SeqId_4_ to hdfs://localhost:38657/hbase/archive/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/2879cc1df97740fba10fec5af3086cde_SeqId_4_ 2024-11-12T10:30:36,193 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/58e4e4c639dc40dc99a16d56471cfaf9 to hdfs://localhost:38657/hbase/archive/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/a/58e4e4c639dc40dc99a16d56471cfaf9 2024-11-12T10:30:36,193 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.HStore(327): Store=2da37b5e3a18d9194bcfa5eb39a145a4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:36,194 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:36,194 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2da37b5e3a18d9194bcfa5eb39a145a4 columnFamilyName b 2024-11-12T10:30:36,194 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:36,195 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.HStore(327): Store=2da37b5e3a18d9194bcfa5eb39a145a4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:36,195 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:36,195 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2da37b5e3a18d9194bcfa5eb39a145a4 columnFamilyName c 2024-11-12T10:30:36,196 DEBUG [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:36,196 INFO [StoreOpener-2da37b5e3a18d9194bcfa5eb39a145a4-1 {}] regionserver.HStore(327): Store=2da37b5e3a18d9194bcfa5eb39a145a4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:36,196 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:36,197 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:36,198 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:36,198 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/recovered.edits/0000000000000000008 2024-11-12T10:30:36,201 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 2da37b5e3a18d9194bcfa5eb39a145a4 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "2da37b5e3a18d9194bcfa5eb39a145a4" family_name: "a" compaction_input: "58e4e4c639dc40dc99a16d56471cfaf9" compaction_input: "12d7ec5e5905457e9f473f187af4a2ff_SeqId_4_" compaction_input: "2879cc1df97740fba10fec5af3086cde_SeqId_4_" compaction_input: "18b9a6d2c15846ec83a892d52c2ac5ab_SeqId_4_" compaction_output: "39e21b8571904fd783b21bdc4764e24a" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731407431852.2da37b5e3a18d9194bcfa5eb39a145a4." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-12T10:30:36,201 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-12T10:30:36,202 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/recovered.edits/0000000000000000008 2024-11-12T10:30:36,202 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/recovered.edits/0000000000000000008 2024-11-12T10:30:36,203 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:36,204 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:36,204 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T10:30:36,205 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2da37b5e3a18d9194bcfa5eb39a145a4 2024-11-12T10:30:36,207 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testCompactedBulkLoadedFiles/2da37b5e3a18d9194bcfa5eb39a145a4/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T10:30:36,208 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2da37b5e3a18d9194bcfa5eb39a145a4; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61684627, jitterRate=-0.08082742989063263}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T10:30:36,208 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2da37b5e3a18d9194bcfa5eb39a145a4: Writing region info on filesystem at 1731407436167Initializing all the Stores at 1731407436168 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407436168Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407436168Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407436168Cleaning up temporary data from old regions at 1731407436204 (+36 ms)Region opened successfully at 1731407436208 (+4 ms) 2024-11-12T10:30:36,211 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2da37b5e3a18d9194bcfa5eb39a145a4, disabling compactions & flushes 2024-11-12T10:30:36,211 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731407431852.2da37b5e3a18d9194bcfa5eb39a145a4. 2024-11-12T10:30:36,211 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731407431852.2da37b5e3a18d9194bcfa5eb39a145a4. 2024-11-12T10:30:36,211 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731407431852.2da37b5e3a18d9194bcfa5eb39a145a4. after waiting 0 ms 2024-11-12T10:30:36,211 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731407431852.2da37b5e3a18d9194bcfa5eb39a145a4. 2024-11-12T10:30:36,212 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731407431852.2da37b5e3a18d9194bcfa5eb39a145a4. 2024-11-12T10:30:36,212 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2da37b5e3a18d9194bcfa5eb39a145a4: Waiting for close lock at 1731407436211Disabling compacts and flushes for region at 1731407436211Disabling writes for close at 1731407436211Writing region close event to WAL at 1731407436212 (+1 ms)Closed at 1731407436212 2024-11-12T10:30:36,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741929_1108 (size=93) 2024-11-12T10:30:36,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741929_1108 (size=93) 2024-11-12T10:30:36,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741929_1108 (size=93) 2024-11-12T10:30:36,217 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T10:30:36,217 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731407436151) 2024-11-12T10:30:36,230 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=438 (was 425) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1606193570_22 at /127.0.0.1:43720 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1606193570_22 at /127.0.0.1:46832 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (865318088) connection to localhost/127.0.0.1:38657 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:38657 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1342 (was 1260) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=526 (was 572), ProcessCount=11 (was 11), AvailableMemoryMB=7199 (was 7218) 2024-11-12T10:30:36,230 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1342 is superior to 1024 2024-11-12T10:30:36,241 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=438, OpenFileDescriptor=1342, MaxFileDescriptor=1048576, SystemLoadAverage=526, ProcessCount=11, AvailableMemoryMB=7198 2024-11-12T10:30:36,241 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1342 is superior to 1024 2024-11-12T10:30:36,254 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:36,256 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T10:30:36,256 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T10:30:36,259 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-34730053, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/hregion-34730053, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:36,270 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-34730053/hregion-34730053.1731407436259, exclude list is [], retry=0 2024-11-12T10:30:36,272 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:36,272 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:36,272 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:36,274 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-34730053/hregion-34730053.1731407436259 2024-11-12T10:30:36,274 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33367:33367)] 2024-11-12T10:30:36,275 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 674b5f3e15992edee1d84d98098360fe, NAME => 'testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38657/hbase 2024-11-12T10:30:36,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741931_1110 (size=67) 2024-11-12T10:30:36,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741931_1110 (size=67) 2024-11-12T10:30:36,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741931_1110 (size=67) 2024-11-12T10:30:36,287 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:36,288 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,289 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 674b5f3e15992edee1d84d98098360fe columnFamilyName a 2024-11-12T10:30:36,289 DEBUG [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:36,290 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(327): Store=674b5f3e15992edee1d84d98098360fe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:36,290 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,291 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 674b5f3e15992edee1d84d98098360fe columnFamilyName b 2024-11-12T10:30:36,291 DEBUG [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:36,292 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(327): Store=674b5f3e15992edee1d84d98098360fe/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:36,292 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,293 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 674b5f3e15992edee1d84d98098360fe columnFamilyName c 2024-11-12T10:30:36,293 DEBUG [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:36,294 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(327): Store=674b5f3e15992edee1d84d98098360fe/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:36,294 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,295 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,295 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,296 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,296 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,297 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T10:30:36,298 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,300 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T10:30:36,301 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 674b5f3e15992edee1d84d98098360fe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73178508, jitterRate=0.09044474363327026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T10:30:36,301 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 674b5f3e15992edee1d84d98098360fe: Writing region info on filesystem at 1731407436287Initializing all the Stores at 1731407436288 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407436288Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407436288Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407436288Cleaning up temporary data from old regions at 1731407436296 (+8 ms)Region opened successfully at 1731407436301 (+5 ms) 2024-11-12T10:30:36,301 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 674b5f3e15992edee1d84d98098360fe, disabling compactions & flushes 2024-11-12T10:30:36,301 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe. 2024-11-12T10:30:36,301 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe. 2024-11-12T10:30:36,301 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe. after waiting 0 ms 2024-11-12T10:30:36,301 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe. 2024-11-12T10:30:36,302 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe. 2024-11-12T10:30:36,302 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 674b5f3e15992edee1d84d98098360fe: Waiting for close lock at 1731407436301Disabling compacts and flushes for region at 1731407436301Disabling writes for close at 1731407436301Writing region close event to WAL at 1731407436302 (+1 ms)Closed at 1731407436302 2024-11-12T10:30:36,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741930_1109 (size=93) 2024-11-12T10:30:36,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741930_1109 (size=93) 2024-11-12T10:30:36,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741930_1109 (size=93) 2024-11-12T10:30:36,306 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T10:30:36,306 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-34730053:(num 1731407436259) 2024-11-12T10:30:36,307 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:36,308 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:36,326 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436309, exclude list is [], retry=0 2024-11-12T10:30:36,328 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:36,329 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:36,329 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:36,330 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436309 2024-11-12T10:30:36,330 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33413:33413)] 2024-11-12T10:30:36,331 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 674b5f3e15992edee1d84d98098360fe, NAME => 'testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:36,331 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:36,331 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,331 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,333 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,334 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 674b5f3e15992edee1d84d98098360fe columnFamilyName a 2024-11-12T10:30:36,334 DEBUG [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:36,334 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(327): Store=674b5f3e15992edee1d84d98098360fe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:36,334 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,335 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 674b5f3e15992edee1d84d98098360fe columnFamilyName b 2024-11-12T10:30:36,335 DEBUG [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:36,335 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(327): Store=674b5f3e15992edee1d84d98098360fe/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:36,336 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,336 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 674b5f3e15992edee1d84d98098360fe columnFamilyName c 2024-11-12T10:30:36,336 DEBUG [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:36,336 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(327): Store=674b5f3e15992edee1d84d98098360fe/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:36,337 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,337 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,338 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,339 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,339 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,340 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T10:30:36,341 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,342 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 674b5f3e15992edee1d84d98098360fe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70429052, jitterRate=0.04947465658187866}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T10:30:36,342 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 674b5f3e15992edee1d84d98098360fe: Writing region info on filesystem at 1731407436331Initializing all the Stores at 1731407436332 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407436332Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407436333 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407436333Cleaning up temporary data from old regions at 1731407436339 (+6 ms)Region opened successfully at 1731407436342 (+3 ms) 2024-11-12T10:30:36,350 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 674b5f3e15992edee1d84d98098360fe 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-12T10:30:36,368 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/.tmp/a/63aa51c56b1a465b809fc0a5ac7f5950 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731407436342/Put/seqid=0 2024-11-12T10:30:36,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741933_1112 (size=5958) 2024-11-12T10:30:36,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741933_1112 (size=5958) 2024-11-12T10:30:36,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741933_1112 (size=5958) 2024-11-12T10:30:36,538 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T10:30:36,777 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/.tmp/a/63aa51c56b1a465b809fc0a5ac7f5950 2024-11-12T10:30:36,788 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/.tmp/a/63aa51c56b1a465b809fc0a5ac7f5950 as hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/a/63aa51c56b1a465b809fc0a5ac7f5950 2024-11-12T10:30:36,795 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/a/63aa51c56b1a465b809fc0a5ac7f5950, entries=10, sequenceid=13, filesize=5.8 K 2024-11-12T10:30:36,796 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 674b5f3e15992edee1d84d98098360fe in 446ms, sequenceid=13, compaction requested=false 2024-11-12T10:30:36,796 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 674b5f3e15992edee1d84d98098360fe: 2024-11-12T10:30:36,814 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 674b5f3e15992edee1d84d98098360fe, disabling compactions & flushes 2024-11-12T10:30:36,814 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe. 2024-11-12T10:30:36,814 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe. 2024-11-12T10:30:36,814 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe. after waiting 0 ms 2024-11-12T10:30:36,814 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe. 2024-11-12T10:30:36,815 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe. 2024-11-12T10:30:36,815 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe. 2024-11-12T10:30:36,815 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 674b5f3e15992edee1d84d98098360fe: Waiting for close lock at 1731407436814Disabling compacts and flushes for region at 1731407436814Disabling writes for close at 1731407436814Writing region close event to WAL at 1731407436815 (+1 ms)Closed at 1731407436815 2024-11-12T10:30:36,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741932_1111 (size=5869) 2024-11-12T10:30:36,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741932_1111 (size=5869) 2024-11-12T10:30:36,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741932_1111 (size=5869) 2024-11-12T10:30:36,831 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436309, size=5.7 K (5869bytes) 2024-11-12T10:30:36,831 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436309 2024-11-12T10:30:36,831 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436309 after 0ms 2024-11-12T10:30:36,833 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436309 took 2ms 2024-11-12T10:30:36,835 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436309 so closing down 2024-11-12T10:30:36,835 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T10:30:36,836 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731407436309.temp 2024-11-12T10:30:36,837 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000003-wal.1731407436309.temp 2024-11-12T10:30:36,837 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T10:30:36,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741934_1113 (size=5223) 2024-11-12T10:30:36,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741934_1113 (size=5223) 2024-11-12T10:30:36,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741934_1113 (size=5223) 2024-11-12T10:30:36,843 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000003-wal.1731407436309.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-12T10:30:36,844 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000003-wal.1731407436309.temp to hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000035 2024-11-12T10:30:36,845 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 11 ms; skipped=2; WAL=hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436309, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-11-12T10:30:36,845 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436309, journal: Splitting hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436309, size=5.7 K (5869bytes) at 1731407436831Finishing writing output for hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436309 so closing down at 1731407436835 (+4 ms)Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000003-wal.1731407436309.temp at 1731407436837 (+2 ms)3 split writer threads finished at 1731407436837Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000003-wal.1731407436309.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731407436843 (+6 ms)Rename recovered edits hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000003-wal.1731407436309.temp to hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000035 at 1731407436844 (+1 ms)Processed 32 edits across 1 Regions in 11 ms; skipped=2; WAL=hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436309, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1731407436845 (+1 ms) 2024-11-12T10:30:36,846 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436309 to hdfs://localhost:38657/hbase/oldWALs/wal.1731407436309 2024-11-12T10:30:36,847 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000035 2024-11-12T10:30:36,847 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:36,848 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:36,859 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436848, exclude list is [], retry=0 2024-11-12T10:30:36,861 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:36,861 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:36,862 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:36,863 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436848 2024-11-12T10:30:36,863 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33413:33413),(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33367:33367)] 2024-11-12T10:30:36,863 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 674b5f3e15992edee1d84d98098360fe, NAME => 'testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe.', STARTKEY => '', ENDKEY => ''} 2024-11-12T10:30:36,863 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:36,864 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,864 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,865 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,866 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 674b5f3e15992edee1d84d98098360fe columnFamilyName a 2024-11-12T10:30:36,866 DEBUG [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:36,871 DEBUG [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/a/63aa51c56b1a465b809fc0a5ac7f5950 2024-11-12T10:30:36,871 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(327): Store=674b5f3e15992edee1d84d98098360fe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:36,871 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,872 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 674b5f3e15992edee1d84d98098360fe columnFamilyName b 2024-11-12T10:30:36,872 DEBUG [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:36,873 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(327): Store=674b5f3e15992edee1d84d98098360fe/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:36,873 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,873 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 674b5f3e15992edee1d84d98098360fe columnFamilyName c 2024-11-12T10:30:36,873 DEBUG [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:36,874 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(327): Store=674b5f3e15992edee1d84d98098360fe/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:36,874 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,874 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,876 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,876 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000035 2024-11-12T10:30:36,879 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000035 2024-11-12T10:30:36,879 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 674b5f3e15992edee1d84d98098360fe 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-12T10:30:36,892 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/.tmp/b/4c0cc1c6d4d94f24bc5a178eba12a611 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731407436796/Put/seqid=0 2024-11-12T10:30:36,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741936_1115 (size=5958) 2024-11-12T10:30:36,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741936_1115 (size=5958) 2024-11-12T10:30:36,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741936_1115 (size=5958) 2024-11-12T10:30:36,898 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/.tmp/b/4c0cc1c6d4d94f24bc5a178eba12a611 2024-11-12T10:30:36,916 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/.tmp/c/e8c42b68286e468491125e05cb7742ef is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731407436804/Put/seqid=0 2024-11-12T10:30:36,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741937_1116 (size=5958) 2024-11-12T10:30:36,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741937_1116 (size=5958) 2024-11-12T10:30:36,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741937_1116 (size=5958) 2024-11-12T10:30:36,922 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/.tmp/c/e8c42b68286e468491125e05cb7742ef 2024-11-12T10:30:36,927 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/.tmp/b/4c0cc1c6d4d94f24bc5a178eba12a611 as hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/b/4c0cc1c6d4d94f24bc5a178eba12a611 2024-11-12T10:30:36,932 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/b/4c0cc1c6d4d94f24bc5a178eba12a611, entries=10, sequenceid=35, filesize=5.8 K 2024-11-12T10:30:36,933 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/.tmp/c/e8c42b68286e468491125e05cb7742ef as hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/c/e8c42b68286e468491125e05cb7742ef 2024-11-12T10:30:36,938 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/c/e8c42b68286e468491125e05cb7742ef, entries=10, sequenceid=35, filesize=5.8 K 2024-11-12T10:30:36,938 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 674b5f3e15992edee1d84d98098360fe in 59ms, sequenceid=35, compaction requested=false; wal=null 2024-11-12T10:30:36,939 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000035 2024-11-12T10:30:36,940 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,940 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,941 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T10:30:36,942 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:36,944 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-12T10:30:36,945 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 674b5f3e15992edee1d84d98098360fe; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67246359, jitterRate=0.0020488351583480835}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T10:30:36,945 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 674b5f3e15992edee1d84d98098360fe: Writing region info on filesystem at 1731407436864Initializing all the Stores at 1731407436864Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407436864Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407436865 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407436865Obtaining lock to block concurrent updates at 1731407436879 (+14 ms)Preparing flush snapshotting stores in 674b5f3e15992edee1d84d98098360fe at 1731407436879Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731407436879Flushing stores of testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe. at 1731407436879Flushing 674b5f3e15992edee1d84d98098360fe/b: creating writer at 1731407436879Flushing 674b5f3e15992edee1d84d98098360fe/b: appending metadata at 1731407436891 (+12 ms)Flushing 674b5f3e15992edee1d84d98098360fe/b: closing flushed file at 1731407436891Flushing 674b5f3e15992edee1d84d98098360fe/c: creating writer at 1731407436902 (+11 ms)Flushing 674b5f3e15992edee1d84d98098360fe/c: appending metadata at 1731407436915 (+13 ms)Flushing 674b5f3e15992edee1d84d98098360fe/c: closing flushed file at 1731407436915Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b03f531: reopening flushed file at 1731407436926 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ede97bb: reopening flushed file at 1731407436932 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 674b5f3e15992edee1d84d98098360fe in 59ms, sequenceid=35, compaction requested=false; wal=null at 1731407436938 (+6 ms)Cleaning up temporary data from old regions at 1731407436940 (+2 ms)Region opened successfully at 1731407436945 (+5 ms) 2024-11-12T10:30:37,006 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436848, size=0 (0bytes) 2024-11-12T10:30:37,006 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436848 might be still open, length is 0 2024-11-12T10:30:37,006 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436848 2024-11-12T10:30:37,006 WARN [IPC Server handler 2 on default port 38657 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436848 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-12T10:30:37,006 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436848 after 0ms 2024-11-12T10:30:39,724 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:43782 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:35741:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43782 dst: /127.0.0.1:35741 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35741 remote=/127.0.0.1:43782]. Total timeout mills is 60000, 57247 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:39,725 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:35592 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:34283:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35592 dst: /127.0.0.1:34283 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:39,725 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1575395495_22 at /127.0.0.1:46892 [Receiving block BP-663294313-172.17.0.3-1731407397077:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:43905:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46892 dst: /127.0.0.1:43905 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:39,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741935_1117 (size=5215) 2024-11-12T10:30:39,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741935_1117 (size=5215) 2024-11-12T10:30:41,007 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436848 after 4001ms 2024-11-12T10:30:41,012 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436848 took 4006ms 2024-11-12T10:30:41,015 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436848; continuing. 2024-11-12T10:30:41,015 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436848 so closing down 2024-11-12T10:30:41,015 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T10:30:41,018 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731407436848.temp 2024-11-12T10:30:41,020 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000037-wal.1731407436848.temp 2024-11-12T10:30:41,020 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T10:30:41,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741938_1118 (size=5223) 2024-11-12T10:30:41,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741938_1118 (size=5223) 2024-11-12T10:30:41,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741938_1118 (size=5223) 2024-11-12T10:30:41,026 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000037-wal.1731407436848.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-12T10:30:41,027 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000037-wal.1731407436848.temp to hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000066 2024-11-12T10:30:41,027 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 14 ms; skipped=0; WAL=hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436848, size=0, length=0, corrupted=false, cancelled=false 2024-11-12T10:30:41,027 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436848, journal: Splitting hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436848, size=0 (0bytes) at 1731407437006Finishing writing output for hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436848 so closing down at 1731407441015 (+4009 ms)Creating recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000037-wal.1731407436848.temp at 1731407441020 (+5 ms)3 split writer threads finished at 1731407441020Closed recovered edits writer path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000037-wal.1731407436848.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731407441026 (+6 ms)Rename recovered edits hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000037-wal.1731407436848.temp to hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000066 at 1731407441027 (+1 ms)Processed 30 edits across 1 Regions in 14 ms; skipped=0; WAL=hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436848, size=0, length=0, corrupted=false, cancelled=false at 1731407441027 2024-11-12T10:30:41,029 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436848 to hdfs://localhost:38657/hbase/oldWALs/wal.1731407436848 2024-11-12T10:30:41,030 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000066 2024-11-12T10:30:41,030 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-12T10:30:41,031 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253, archiveDir=hdfs://localhost:38657/hbase/oldWALs, maxLogs=32 2024-11-12T10:30:41,043 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407441031, exclude list is [], retry=0 2024-11-12T10:30:41,045 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43905,DS-90f90408-e2b3-455c-9019-25fea3fabe0e,DISK] 2024-11-12T10:30:41,045 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34283,DS-b693e0e9-f5af-43e6-abcd-1f6a2023b98b,DISK] 2024-11-12T10:30:41,045 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35741,DS-d76d6edc-a670-416d-b284-2cd56e55dc61,DISK] 2024-11-12T10:30:41,047 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407441031 2024-11-12T10:30:41,047 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39823:39823),(127.0.0.1/127.0.0.1:33367:33367),(127.0.0.1/127.0.0.1:33413:33413)] 2024-11-12T10:30:41,048 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T10:30:41,049 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:41,050 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 674b5f3e15992edee1d84d98098360fe columnFamilyName a 2024-11-12T10:30:41,050 DEBUG [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:41,054 DEBUG [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/a/63aa51c56b1a465b809fc0a5ac7f5950 2024-11-12T10:30:41,055 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(327): Store=674b5f3e15992edee1d84d98098360fe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:41,055 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:41,056 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 674b5f3e15992edee1d84d98098360fe columnFamilyName b 2024-11-12T10:30:41,056 DEBUG [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:41,061 DEBUG [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/b/4c0cc1c6d4d94f24bc5a178eba12a611 2024-11-12T10:30:41,061 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(327): Store=674b5f3e15992edee1d84d98098360fe/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:41,061 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:41,062 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 674b5f3e15992edee1d84d98098360fe columnFamilyName c 2024-11-12T10:30:41,062 DEBUG [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T10:30:41,068 DEBUG [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/c/e8c42b68286e468491125e05cb7742ef 2024-11-12T10:30:41,068 INFO [StoreOpener-674b5f3e15992edee1d84d98098360fe-1 {}] regionserver.HStore(327): Store=674b5f3e15992edee1d84d98098360fe/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T10:30:41,068 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:41,069 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:41,071 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:41,072 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000066 2024-11-12T10:30:41,080 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000066 2024-11-12T10:30:41,080 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 674b5f3e15992edee1d84d98098360fe 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-12T10:30:41,094 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/.tmp/a/85af68e6fe4248e3b6ccb0c16a431f66 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731407436951/Put/seqid=0 2024-11-12T10:30:41,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741940_1120 (size=5958) 2024-11-12T10:30:41,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741940_1120 (size=5958) 2024-11-12T10:30:41,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741940_1120 (size=5958) 2024-11-12T10:30:41,101 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/.tmp/a/85af68e6fe4248e3b6ccb0c16a431f66 2024-11-12T10:30:41,120 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/.tmp/b/811967df806f49039833b5d990f41d69 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731407436958/Put/seqid=0 2024-11-12T10:30:41,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741941_1121 (size=5958) 2024-11-12T10:30:41,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741941_1121 (size=5958) 2024-11-12T10:30:41,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741941_1121 (size=5958) 2024-11-12T10:30:41,126 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/.tmp/b/811967df806f49039833b5d990f41d69 2024-11-12T10:30:41,142 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/.tmp/c/f1f6bd4f575449b6929d8f4e12e7b004 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731407436964/Put/seqid=0 2024-11-12T10:30:41,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741942_1122 (size=5958) 2024-11-12T10:30:41,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741942_1122 (size=5958) 2024-11-12T10:30:41,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741942_1122 (size=5958) 2024-11-12T10:30:41,149 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/.tmp/c/f1f6bd4f575449b6929d8f4e12e7b004 2024-11-12T10:30:41,153 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/.tmp/a/85af68e6fe4248e3b6ccb0c16a431f66 as hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/a/85af68e6fe4248e3b6ccb0c16a431f66 2024-11-12T10:30:41,158 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/a/85af68e6fe4248e3b6ccb0c16a431f66, entries=10, sequenceid=66, filesize=5.8 K 2024-11-12T10:30:41,159 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/.tmp/b/811967df806f49039833b5d990f41d69 as hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/b/811967df806f49039833b5d990f41d69 2024-11-12T10:30:41,163 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/b/811967df806f49039833b5d990f41d69, entries=10, sequenceid=66, filesize=5.8 K 2024-11-12T10:30:41,164 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/.tmp/c/f1f6bd4f575449b6929d8f4e12e7b004 as hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/c/f1f6bd4f575449b6929d8f4e12e7b004 2024-11-12T10:30:41,169 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/c/f1f6bd4f575449b6929d8f4e12e7b004, entries=10, sequenceid=66, filesize=5.8 K 2024-11-12T10:30:41,169 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 674b5f3e15992edee1d84d98098360fe in 89ms, sequenceid=66, compaction requested=false; wal=null 2024-11-12T10:30:41,170 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/0000000000000000066 2024-11-12T10:30:41,171 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:41,171 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:41,172 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T10:30:41,173 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 674b5f3e15992edee1d84d98098360fe 2024-11-12T10:30:41,175 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/hbase/data/default/testReplayEditsWrittenViaHRegion/674b5f3e15992edee1d84d98098360fe/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-12T10:30:41,176 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 674b5f3e15992edee1d84d98098360fe; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66215616, jitterRate=-0.013310432434082031}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T10:30:41,176 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 674b5f3e15992edee1d84d98098360fe: Writing region info on filesystem at 1731407441048Initializing all the Stores at 1731407441048Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407441048Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407441049 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731407441049Obtaining lock to block concurrent updates at 1731407441080 (+31 ms)Preparing flush snapshotting stores in 674b5f3e15992edee1d84d98098360fe at 1731407441080Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731407441080Flushing stores of testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe. at 1731407441080Flushing 674b5f3e15992edee1d84d98098360fe/a: creating writer at 1731407441080Flushing 674b5f3e15992edee1d84d98098360fe/a: appending metadata at 1731407441093 (+13 ms)Flushing 674b5f3e15992edee1d84d98098360fe/a: closing flushed file at 1731407441093Flushing 674b5f3e15992edee1d84d98098360fe/b: creating writer at 1731407441106 (+13 ms)Flushing 674b5f3e15992edee1d84d98098360fe/b: appending metadata at 1731407441119 (+13 ms)Flushing 674b5f3e15992edee1d84d98098360fe/b: closing flushed file at 1731407441119Flushing 674b5f3e15992edee1d84d98098360fe/c: creating writer at 1731407441129 (+10 ms)Flushing 674b5f3e15992edee1d84d98098360fe/c: appending metadata at 1731407441142 (+13 ms)Flushing 674b5f3e15992edee1d84d98098360fe/c: closing flushed file at 1731407441142Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53f40c86: reopening flushed file at 1731407441153 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@723a6bdb: reopening flushed file at 1731407441158 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39a4d7ba: reopening flushed file at 1731407441163 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 674b5f3e15992edee1d84d98098360fe in 89ms, sequenceid=66, compaction requested=false; wal=null at 1731407441170 (+7 ms)Cleaning up temporary data from old regions at 1731407441171 (+1 ms)Region opened successfully at 1731407441176 (+5 ms) 2024-11-12T10:30:41,186 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 674b5f3e15992edee1d84d98098360fe, disabling compactions & flushes 2024-11-12T10:30:41,186 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe. 2024-11-12T10:30:41,186 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe. 2024-11-12T10:30:41,187 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe. after waiting 0 ms 2024-11-12T10:30:41,187 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe. 2024-11-12T10:30:41,188 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731407436254.674b5f3e15992edee1d84d98098360fe. 2024-11-12T10:30:41,188 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 674b5f3e15992edee1d84d98098360fe: Waiting for close lock at 1731407441186Disabling compacts and flushes for region at 1731407441186Disabling writes for close at 1731407441187 (+1 ms)Writing region close event to WAL at 1731407441188 (+1 ms)Closed at 1731407441188 2024-11-12T10:30:41,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741939_1119 (size=93) 2024-11-12T10:30:41,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741939_1119 (size=93) 2024-11-12T10:30:41,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741939_1119 (size=93) 2024-11-12T10:30:41,192 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-12T10:30:41,192 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731407441031) 2024-11-12T10:30:41,207 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=444 (was 438) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1915205720_22 at /127.0.0.1:43804 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (865318088) connection to localhost/127.0.0.1:38657 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1915205720_22 at /127.0.0.1:35652 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1915205720_22 at /127.0.0.1:46968 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:38657 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1410 (was 1342) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=484 (was 526), ProcessCount=11 (was 11), AvailableMemoryMB=7167 (was 7198) 2024-11-12T10:30:41,207 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1410 is superior to 1024 2024-11-12T10:30:41,208 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T10:30:41,208 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T10:30:41,208 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T10:30:41,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:30:41,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:30:41,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T10:30:41,209 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T10:30:41,209 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1885054661, stopped=false 2024-11-12T10:30:41,209 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a10fb4296bab,35883,1731407401663 2024-11-12T10:30:41,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T10:30:41,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T10:30:41,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T10:30:41,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:41,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:41,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:41,270 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T10:30:41,270 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T10:30:41,271 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T10:30:41,271 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T10:30:41,271 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:30:41,271 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a10fb4296bab,45327,1731407402438' ***** 2024-11-12T10:30:41,271 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T10:30:41,271 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a10fb4296bab,40713,1731407402577' ***** 2024-11-12T10:30:41,271 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T10:30:41,271 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T10:30:41,271 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T10:30:41,271 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T10:30:41,271 INFO [RS:2;a10fb4296bab:40713 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T10:30:41,272 INFO [RS:0;a10fb4296bab:45327 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T10:30:41,272 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T10:30:41,272 INFO [RS:0;a10fb4296bab:45327 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T10:30:41,272 INFO [RS:2;a10fb4296bab:40713 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T10:30:41,272 INFO [RS:2;a10fb4296bab:40713 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T10:30:41,272 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.HRegionServer(3091): Received CLOSE for eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:41,272 INFO [RS:2;a10fb4296bab:40713 {}] regionserver.HRegionServer(959): stopping server a10fb4296bab,40713,1731407402577 2024-11-12T10:30:41,272 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.HRegionServer(959): stopping server a10fb4296bab,45327,1731407402438 2024-11-12T10:30:41,272 INFO [RS:2;a10fb4296bab:40713 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T10:30:41,272 INFO [RS:0;a10fb4296bab:45327 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T10:30:41,272 INFO [RS:2;a10fb4296bab:40713 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;a10fb4296bab:40713. 2024-11-12T10:30:41,272 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eaa006ae0460d9d2606f62df60fe50d7, disabling compactions & flushes 2024-11-12T10:30:41,272 INFO [RS:0;a10fb4296bab:45327 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a10fb4296bab:45327. 2024-11-12T10:30:41,272 DEBUG [RS:2;a10fb4296bab:40713 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T10:30:41,272 INFO [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:41,272 DEBUG [RS:2;a10fb4296bab:40713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:30:41,273 DEBUG [RS:0;a10fb4296bab:45327 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T10:30:41,273 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:41,273 DEBUG [RS:0;a10fb4296bab:45327 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:30:41,273 INFO [RS:2;a10fb4296bab:40713 {}] regionserver.HRegionServer(976): stopping server a10fb4296bab,40713,1731407402577; all regions closed. 2024-11-12T10:30:41,273 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. after waiting 0 ms 2024-11-12T10:30:41,273 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T10:30:41,273 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T10:30:41,273 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:41,273 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T10:30:41,273 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T10:30:41,273 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T10:30:41,274 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-12T10:30:41,274 DEBUG [RS:0;a10fb4296bab:45327 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, eaa006ae0460d9d2606f62df60fe50d7=testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7.} 2024-11-12T10:30:41,274 DEBUG [RS:0;a10fb4296bab:45327 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, eaa006ae0460d9d2606f62df60fe50d7 2024-11-12T10:30:41,274 DEBUG [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T10:30:41,274 INFO [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T10:30:41,274 DEBUG [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T10:30:41,274 DEBUG [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T10:30:41,274 DEBUG [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T10:30:41,275 INFO [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-12T10:30:41,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741833_1009 (size=93) 2024-11-12T10:30:41,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741833_1009 (size=93) 2024-11-12T10:30:41,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741833_1009 (size=93) 2024-11-12T10:30:41,281 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/default/testReplayEditsAfterRegionMovedWithMultiCF/eaa006ae0460d9d2606f62df60fe50d7/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-12T10:30:41,281 DEBUG [RS:2;a10fb4296bab:40713 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/oldWALs 2024-11-12T10:30:41,281 INFO [RS:2;a10fb4296bab:40713 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a10fb4296bab%2C40713%2C1731407402577:(num 1731407404072) 2024-11-12T10:30:41,281 DEBUG [RS:2;a10fb4296bab:40713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:30:41,281 INFO [RS:2;a10fb4296bab:40713 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T10:30:41,281 INFO [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:41,281 INFO [RS:2;a10fb4296bab:40713 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T10:30:41,282 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eaa006ae0460d9d2606f62df60fe50d7: Waiting for close lock at 1731407441272Running coprocessor pre-close hooks at 1731407441272Disabling compacts and flushes for region at 1731407441272Disabling writes for close at 1731407441273 (+1 ms)Writing region close event to WAL at 1731407441275 (+2 ms)Running coprocessor post-close hooks at 1731407441281 (+6 ms)Closed at 1731407441281 2024-11-12T10:30:41,282 INFO [RS:2;a10fb4296bab:40713 {}] hbase.ChoreService(370): Chore service for: regionserver/a10fb4296bab:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T10:30:41,282 DEBUG [RS_CLOSE_REGION-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7. 2024-11-12T10:30:41,282 INFO [RS:2;a10fb4296bab:40713 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T10:30:41,282 INFO [RS:2;a10fb4296bab:40713 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T10:30:41,282 INFO [regionserver/a10fb4296bab:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T10:30:41,282 INFO [RS:2;a10fb4296bab:40713 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T10:30:41,282 INFO [RS:2;a10fb4296bab:40713 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T10:30:41,282 INFO [RS:2;a10fb4296bab:40713 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40713 2024-11-12T10:30:41,283 DEBUG [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/.tmp/info/4327e89b89724ffab4aad770d41df35d is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731407416743.eaa006ae0460d9d2606f62df60fe50d7./info:regioninfo/1731407421040/Put/seqid=0 2024-11-12T10:30:41,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741943_1123 (size=6778) 2024-11-12T10:30:41,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741943_1123 (size=6778) 2024-11-12T10:30:41,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741943_1123 (size=6778) 2024-11-12T10:30:41,289 INFO [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/.tmp/info/4327e89b89724ffab4aad770d41df35d 2024-11-12T10:30:41,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a10fb4296bab,40713,1731407402577 2024-11-12T10:30:41,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T10:30:41,291 INFO [RS:2;a10fb4296bab:40713 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T10:30:41,291 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a10fb4296bab,40713,1731407402577] 2024-11-12T10:30:41,294 DEBUG [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/.tmp/info/4327e89b89724ffab4aad770d41df35d as hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/info/4327e89b89724ffab4aad770d41df35d 2024-11-12T10:30:41,298 INFO [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/info/4327e89b89724ffab4aad770d41df35d, entries=8, sequenceid=23, filesize=6.6 K 2024-11-12T10:30:41,299 INFO [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 25ms, sequenceid=23, compaction requested=false 2024-11-12T10:30:41,299 DEBUG [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T10:30:41,304 DEBUG [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-12T10:30:41,305 DEBUG [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T10:30:41,305 INFO [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T10:30:41,305 DEBUG [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731407441274Running coprocessor pre-close hooks at 1731407441274Disabling compacts and flushes for region at 1731407441274Disabling writes for close at 1731407441274Obtaining lock to block concurrent updates at 1731407441275 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731407441275Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1731407441275Flushing stores of hbase:meta,,1.1588230740 at 1731407441278 (+3 ms)Flushing 1588230740/info: creating writer at 1731407441279 (+1 ms)Flushing 1588230740/info: appending metadata at 1731407441282 (+3 ms)Flushing 1588230740/info: closing flushed file at 1731407441283 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bc8984a: reopening flushed file at 1731407441293 (+10 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 25ms, sequenceid=23, compaction requested=false at 1731407441299 (+6 ms)Writing region close event to WAL at 1731407441301 (+2 ms)Running coprocessor post-close hooks at 1731407441305 (+4 ms)Closed at 1731407441305 2024-11-12T10:30:41,305 DEBUG [RS_CLOSE_META-regionserver/a10fb4296bab:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T10:30:41,312 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a10fb4296bab,40713,1731407402577 already deleted, retry=false 2024-11-12T10:30:41,312 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a10fb4296bab,40713,1731407402577 expired; onlineServers=1 2024-11-12T10:30:41,361 INFO [regionserver/a10fb4296bab:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T10:30:41,361 INFO [regionserver/a10fb4296bab:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T10:30:41,401 INFO [RS:2;a10fb4296bab:40713 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T10:30:41,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T10:30:41,402 INFO [RS:2;a10fb4296bab:40713 {}] regionserver.HRegionServer(1031): Exiting; stopping=a10fb4296bab,40713,1731407402577; zookeeper connection closed. 2024-11-12T10:30:41,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40713-0x1012e984b560003, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T10:30:41,402 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@b38488a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@b38488a 2024-11-12T10:30:41,474 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.HRegionServer(976): stopping server a10fb4296bab,45327,1731407402438; all regions closed. 2024-11-12T10:30:41,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741893_1071 (size=2422) 2024-11-12T10:30:41,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741893_1071 (size=2422) 2024-11-12T10:30:41,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741893_1071 (size=2422) 2024-11-12T10:30:41,485 DEBUG [RS:0;a10fb4296bab:45327 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/oldWALs 2024-11-12T10:30:41,485 INFO [RS:0;a10fb4296bab:45327 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a10fb4296bab%2C45327%2C1731407402438.meta:.meta(num 1731407419996) 2024-11-12T10:30:41,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741835_1011 (size=3433) 2024-11-12T10:30:41,487 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/WALs/a10fb4296bab,45327,1731407402438/a10fb4296bab%2C45327%2C1731407402438.1731407404072 not finished, retry = 0 2024-11-12T10:30:41,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741835_1011 (size=3433) 2024-11-12T10:30:41,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741835_1011 (size=3433) 2024-11-12T10:30:41,592 DEBUG [RS:0;a10fb4296bab:45327 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/oldWALs 2024-11-12T10:30:41,592 INFO [RS:0;a10fb4296bab:45327 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a10fb4296bab%2C45327%2C1731407402438:(num 1731407404072) 2024-11-12T10:30:41,592 DEBUG [RS:0;a10fb4296bab:45327 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T10:30:41,592 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T10:30:41,593 INFO [RS:0;a10fb4296bab:45327 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T10:30:41,593 INFO [RS:0;a10fb4296bab:45327 {}] hbase.ChoreService(370): Chore service for: regionserver/a10fb4296bab:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T10:30:41,593 INFO [RS:0;a10fb4296bab:45327 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T10:30:41,593 INFO [regionserver/a10fb4296bab:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T10:30:41,594 INFO [RS:0;a10fb4296bab:45327 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45327 2024-11-12T10:30:41,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a10fb4296bab,45327,1731407402438 2024-11-12T10:30:41,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T10:30:41,625 INFO [RS:0;a10fb4296bab:45327 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T10:30:41,639 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a10fb4296bab,45327,1731407402438] 2024-11-12T10:30:41,649 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a10fb4296bab,45327,1731407402438 already deleted, retry=false 2024-11-12T10:30:41,649 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a10fb4296bab,45327,1731407402438 expired; onlineServers=0 2024-11-12T10:30:41,650 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a10fb4296bab,35883,1731407401663' ***** 2024-11-12T10:30:41,650 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T10:30:41,650 INFO [M:0;a10fb4296bab:35883 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T10:30:41,651 INFO [M:0;a10fb4296bab:35883 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T10:30:41,651 DEBUG [M:0;a10fb4296bab:35883 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T10:30:41,651 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T10:30:41,652 DEBUG [M:0;a10fb4296bab:35883 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T10:30:41,652 DEBUG [master/a10fb4296bab:0:becomeActiveMaster-HFileCleaner.large.0-1731407403726 {}] cleaner.HFileCleaner(306): Exit Thread[master/a10fb4296bab:0:becomeActiveMaster-HFileCleaner.large.0-1731407403726,5,FailOnTimeoutGroup] 2024-11-12T10:30:41,652 DEBUG [master/a10fb4296bab:0:becomeActiveMaster-HFileCleaner.small.0-1731407403727 {}] cleaner.HFileCleaner(306): Exit Thread[master/a10fb4296bab:0:becomeActiveMaster-HFileCleaner.small.0-1731407403727,5,FailOnTimeoutGroup] 2024-11-12T10:30:41,652 INFO [M:0;a10fb4296bab:35883 {}] hbase.ChoreService(370): Chore service for: master/a10fb4296bab:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T10:30:41,653 INFO [M:0;a10fb4296bab:35883 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T10:30:41,653 DEBUG [M:0;a10fb4296bab:35883 {}] master.HMaster(1795): Stopping service threads 2024-11-12T10:30:41,653 INFO [M:0;a10fb4296bab:35883 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T10:30:41,654 INFO [M:0;a10fb4296bab:35883 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T10:30:41,655 INFO [M:0;a10fb4296bab:35883 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T10:30:41,655 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T10:30:41,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T10:30:41,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T10:30:41,660 DEBUG [M:0;a10fb4296bab:35883 {}] zookeeper.ZKUtil(347): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-12T10:30:41,660 WARN [M:0;a10fb4296bab:35883 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T10:30:41,661 INFO [M:0;a10fb4296bab:35883 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/.lastflushedseqids 2024-11-12T10:30:41,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741944_1124 (size=119) 2024-11-12T10:30:41,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741944_1124 (size=119) 2024-11-12T10:30:41,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741944_1124 (size=119) 2024-11-12T10:30:41,675 INFO [M:0;a10fb4296bab:35883 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T10:30:41,675 INFO [M:0;a10fb4296bab:35883 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T10:30:41,676 DEBUG [M:0;a10fb4296bab:35883 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T10:30:41,676 INFO [M:0;a10fb4296bab:35883 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T10:30:41,676 DEBUG [M:0;a10fb4296bab:35883 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T10:30:41,676 DEBUG [M:0;a10fb4296bab:35883 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T10:30:41,676 DEBUG [M:0;a10fb4296bab:35883 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T10:30:41,676 INFO [M:0;a10fb4296bab:35883 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.40 KB heapSize=102.70 KB 2024-11-12T10:30:41,691 DEBUG [M:0;a10fb4296bab:35883 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bb86c9f7094747d3bff6bdebcdaa5f4f is 82, key is hbase:meta,,1/info:regioninfo/1731407420139/Put/seqid=0 2024-11-12T10:30:41,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741945_1125 (size=6063) 2024-11-12T10:30:41,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741945_1125 (size=6063) 2024-11-12T10:30:41,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741945_1125 (size=6063) 2024-11-12T10:30:41,697 INFO [M:0;a10fb4296bab:35883 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bb86c9f7094747d3bff6bdebcdaa5f4f 2024-11-12T10:30:41,714 DEBUG [M:0;a10fb4296bab:35883 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6020a16f9b85468bb7346a59f4a95f6c is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731407417184/Put/seqid=0 2024-11-12T10:30:41,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741946_1126 (size=7907) 2024-11-12T10:30:41,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741946_1126 (size=7907) 2024-11-12T10:30:41,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741946_1126 (size=7907) 2024-11-12T10:30:41,720 INFO [M:0;a10fb4296bab:35883 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.17 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6020a16f9b85468bb7346a59f4a95f6c 2024-11-12T10:30:41,724 INFO [M:0;a10fb4296bab:35883 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6020a16f9b85468bb7346a59f4a95f6c 2024-11-12T10:30:41,737 DEBUG [M:0;a10fb4296bab:35883 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/518f3e16acbc4d10a42a44559144e914 is 69, key is a10fb4296bab,40713,1731407402577/rs:state/1731407403763/Put/seqid=0 2024-11-12T10:30:41,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T10:30:41,739 INFO [RS:0;a10fb4296bab:45327 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T10:30:41,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45327-0x1012e984b560001, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T10:30:41,739 INFO [RS:0;a10fb4296bab:45327 {}] regionserver.HRegionServer(1031): Exiting; stopping=a10fb4296bab,45327,1731407402438; zookeeper connection closed. 2024-11-12T10:30:41,739 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5140432d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5140432d 2024-11-12T10:30:41,739 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-12T10:30:41,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741947_1127 (size=5440) 2024-11-12T10:30:41,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741947_1127 (size=5440) 2024-11-12T10:30:41,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741947_1127 (size=5440) 2024-11-12T10:30:41,744 INFO [M:0;a10fb4296bab:35883 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/518f3e16acbc4d10a42a44559144e914 2024-11-12T10:30:41,749 INFO [M:0;a10fb4296bab:35883 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 518f3e16acbc4d10a42a44559144e914 2024-11-12T10:30:41,750 DEBUG [M:0;a10fb4296bab:35883 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bb86c9f7094747d3bff6bdebcdaa5f4f as hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bb86c9f7094747d3bff6bdebcdaa5f4f 2024-11-12T10:30:41,756 INFO [M:0;a10fb4296bab:35883 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bb86c9f7094747d3bff6bdebcdaa5f4f, entries=14, sequenceid=207, filesize=5.9 K 2024-11-12T10:30:41,757 DEBUG [M:0;a10fb4296bab:35883 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6020a16f9b85468bb7346a59f4a95f6c as hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6020a16f9b85468bb7346a59f4a95f6c 2024-11-12T10:30:41,762 INFO [M:0;a10fb4296bab:35883 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6020a16f9b85468bb7346a59f4a95f6c 2024-11-12T10:30:41,762 INFO [M:0;a10fb4296bab:35883 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6020a16f9b85468bb7346a59f4a95f6c, entries=21, sequenceid=207, filesize=7.7 K 2024-11-12T10:30:41,763 DEBUG [M:0;a10fb4296bab:35883 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/518f3e16acbc4d10a42a44559144e914 as hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/518f3e16acbc4d10a42a44559144e914 2024-11-12T10:30:41,768 INFO [M:0;a10fb4296bab:35883 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 518f3e16acbc4d10a42a44559144e914 2024-11-12T10:30:41,768 INFO [M:0;a10fb4296bab:35883 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38657/user/jenkins/test-data/490110b1-bce7-bed6-fecd-0171db83cacf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/518f3e16acbc4d10a42a44559144e914, entries=3, sequenceid=207, filesize=5.3 K 2024-11-12T10:30:41,769 INFO [M:0;a10fb4296bab:35883 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 93ms, sequenceid=207, compaction requested=false 2024-11-12T10:30:41,771 INFO [M:0;a10fb4296bab:35883 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T10:30:41,771 DEBUG [M:0;a10fb4296bab:35883 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731407441676Disabling compacts and flushes for region at 1731407441676Disabling writes for close at 1731407441676Obtaining lock to block concurrent updates at 1731407441676Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731407441676Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85398, getHeapSize=105104, getOffHeapSize=0, getCellsCount=248 at 1731407441676Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731407441677 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731407441677Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731407441691 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731407441691Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731407441701 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731407441714 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731407441714Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731407441724 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731407441737 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731407441737Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b514596: reopening flushed file at 1731407441749 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b60c9cc: reopening flushed file at 1731407441756 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1358bb04: reopening flushed file at 1731407441762 (+6 ms)Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 93ms, sequenceid=207, compaction requested=false at 1731407441769 (+7 ms)Writing region close event to WAL at 1731407441771 (+2 ms)Closed at 1731407441771 2024-11-12T10:30:41,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43905 is added to blk_1073741830_1006 (size=99436) 2024-11-12T10:30:41,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35741 is added to blk_1073741830_1006 (size=99436) 2024-11-12T10:30:41,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34283 is added to blk_1073741830_1006 (size=99436) 2024-11-12T10:30:41,775 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T10:30:41,775 INFO [M:0;a10fb4296bab:35883 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T10:30:41,776 INFO [M:0;a10fb4296bab:35883 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35883 2024-11-12T10:30:41,776 INFO [M:0;a10fb4296bab:35883 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T10:30:41,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T10:30:41,886 INFO [M:0;a10fb4296bab:35883 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T10:30:41,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35883-0x1012e984b560000, quorum=127.0.0.1:55409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T10:30:41,897 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829/wal.1731407431051 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829/wal.1731407431051 (inode 16688) Holder DFSClient_NONMAPREDUCE_1575395495_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731407421829/wal.1731407431051 (inode 16688) Holder DFSClient_NONMAPREDUCE_1575395495_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-12T10:30:41,901 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400/wal.1731407421680 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400/wal.1731407421680 (inode 16665) Holder DFSClient_NONMAPREDUCE_1575395495_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731407421400/wal.1731407421680 (inode 16665) Holder DFSClient_NONMAPREDUCE_1575395495_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-12T10:30:41,902 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731407412342/wal.1731407412439 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:41,904 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232/wal.1731407431712 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232/wal.1731407431712 (inode 16714) Holder DFSClient_NONMAPREDUCE_1575395495_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731407431232/wal.1731407431712 (inode 16714) Holder DFSClient_NONMAPREDUCE_1575395495_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-12T10:30:41,905 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731407431851/wal.1731407431916 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:41,905 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731407436253/wal.1731407436848 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:41,905 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731407405832/wal.1731407405905 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T10:30:41,908 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal.1731407405712 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal.1731407405712 (inode 16485) Holder DFSClient_NONMAPREDUCE_1575395495_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731407405489/wal.1731407405712 (inode 16485) Holder DFSClient_NONMAPREDUCE_1575395495_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-12T10:30:41,911 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal.1731407405197 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal.1731407405197 (inode 16462) Holder DFSClient_NONMAPREDUCE_1575395495_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731407404922/wal.1731407405197 (inode 16462) Holder DFSClient_NONMAPREDUCE_1575395495_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-12T10:30:41,915 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@50b13206{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T10:30:41,916 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3267e685{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T10:30:41,917 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T10:30:41,917 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f8696d2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T10:30:41,917 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@37d4eea4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/hadoop.log.dir/,STOPPED} 2024-11-12T10:30:41,919 WARN [BP-663294313-172.17.0.3-1731407397077 heartbeating to localhost/127.0.0.1:38657 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T10:30:41,919 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T10:30:41,919 WARN [BP-663294313-172.17.0.3-1731407397077 heartbeating to localhost/127.0.0.1:38657 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-663294313-172.17.0.3-1731407397077 (Datanode Uuid be38b4fe-5f2b-4421-a282-3500d61d9da1) service to localhost/127.0.0.1:38657 2024-11-12T10:30:41,919 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T10:30:41,920 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/data/data5/current/BP-663294313-172.17.0.3-1731407397077 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T10:30:41,920 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/data/data6/current/BP-663294313-172.17.0.3-1731407397077 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T10:30:41,921 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T10:30:41,927 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@494486cb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T10:30:41,927 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7d076b20{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T10:30:41,927 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T10:30:41,927 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@152e2397{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T10:30:41,928 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fc499ca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/hadoop.log.dir/,STOPPED} 2024-11-12T10:30:41,929 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T10:30:41,929 WARN [BP-663294313-172.17.0.3-1731407397077 heartbeating to localhost/127.0.0.1:38657 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T10:30:41,929 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T10:30:41,929 WARN [BP-663294313-172.17.0.3-1731407397077 heartbeating to localhost/127.0.0.1:38657 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-663294313-172.17.0.3-1731407397077 (Datanode Uuid 45d0bcac-c90a-4a24-9c79-0339c85d65f1) service to localhost/127.0.0.1:38657 2024-11-12T10:30:41,929 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/data/data3/current/BP-663294313-172.17.0.3-1731407397077 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T10:30:41,930 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/data/data4/current/BP-663294313-172.17.0.3-1731407397077 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T10:30:41,930 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T10:30:41,937 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5bb7bef3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T10:30:41,937 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4080a860{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T10:30:41,937 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T10:30:41,938 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4c19a0d8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T10:30:41,938 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29f9ab09{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/hadoop.log.dir/,STOPPED} 2024-11-12T10:30:41,939 WARN [BP-663294313-172.17.0.3-1731407397077 heartbeating to localhost/127.0.0.1:38657 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T10:30:41,939 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T10:30:41,939 WARN [BP-663294313-172.17.0.3-1731407397077 heartbeating to localhost/127.0.0.1:38657 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-663294313-172.17.0.3-1731407397077 (Datanode Uuid 702c94b9-c9ff-4905-948b-bb25cc8483bb) service to localhost/127.0.0.1:38657 2024-11-12T10:30:41,939 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T10:30:41,940 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/data/data1/current/BP-663294313-172.17.0.3-1731407397077 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T10:30:41,940 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/cluster_3217c7b8-84ec-bd00-1586-187cad211436/data/data2/current/BP-663294313-172.17.0.3-1731407397077 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T10:30:41,941 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T10:30:41,946 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@398d2d49{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T10:30:41,946 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T10:30:41,946 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T10:30:41,947 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T10:30:41,947 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3e38a5b7-430d-faa0-70d2-28ac28f36445/hadoop.log.dir/,STOPPED} 2024-11-12T10:30:41,955 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T10:30:42,005 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-12T10:30:42,005 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T10:30:42,006 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-12T10:30:42,006 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-12T10:30:42,006 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF