2024-11-27 12:28:02,876 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-27 12:28:02,893 main DEBUG Took 0.013969 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-27 12:28:02,893 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-27 12:28:02,894 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-27 12:28:02,895 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-27 12:28:02,896 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:28:02,905 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-27 12:28:02,916 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:28:02,918 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:28:02,918 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:28:02,919 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:28:02,919 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:28:02,919 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:28:02,920 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:28:02,921 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:28:02,921 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:28:02,921 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:28:02,922 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:28:02,922 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:28:02,923 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:28:02,923 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:28:02,924 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:28:02,924 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:28:02,925 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:28:02,925 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:28:02,925 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:28:02,926 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:28:02,926 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:28:02,926 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:28:02,927 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:28:02,927 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:28:02,927 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:28:02,928 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-27 12:28:02,929 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:28:02,931 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-27 12:28:02,933 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-27 12:28:02,933 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-27 12:28:02,934 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-27 12:28:02,935 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-27 12:28:02,943 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-27 12:28:02,946 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-27 12:28:02,947 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-27 12:28:02,948 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-27 12:28:02,948 main DEBUG createAppenders(={Console}) 2024-11-27 12:28:02,949 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-11-27 12:28:02,949 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-27 12:28:02,950 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-11-27 12:28:02,950 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-27 12:28:02,951 main DEBUG OutputStream closed 2024-11-27 12:28:02,951 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-27 12:28:02,951 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-27 12:28:02,951 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-11-27 12:28:03,025 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-27 12:28:03,027 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-27 12:28:03,028 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-27 12:28:03,030 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-27 12:28:03,030 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-27 12:28:03,031 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-27 12:28:03,031 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-27 12:28:03,031 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-27 12:28:03,032 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-27 12:28:03,032 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-27 12:28:03,032 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-27 12:28:03,033 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-27 12:28:03,033 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-27 12:28:03,033 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-27 12:28:03,033 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-27 12:28:03,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-27 12:28:03,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-27 12:28:03,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-27 12:28:03,037 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27 12:28:03,037 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-11-27 12:28:03,038 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-27 12:28:03,038 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-11-27T12:28:03,290 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5 2024-11-27 12:28:03,292 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-27 12:28:03,293 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27T12:28:03,301 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-27T12:28:03,329 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-27T12:28:03,371 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-27T12:28:03,371 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-27T12:28:03,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-27T12:28:03,397 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401, deleteOnExit=true 2024-11-27T12:28:03,398 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-27T12:28:03,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/test.cache.data in system properties and HBase conf 2024-11-27T12:28:03,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/hadoop.tmp.dir in system properties and HBase conf 2024-11-27T12:28:03,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/hadoop.log.dir in system properties and HBase conf 2024-11-27T12:28:03,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-27T12:28:03,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-27T12:28:03,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-27T12:28:03,477 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-27T12:28:03,569 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:28:03,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-27T12:28:03,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-27T12:28:03,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-27T12:28:03,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T12:28:03,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-27T12:28:03,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-27T12:28:03,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T12:28:03,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T12:28:03,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-27T12:28:03,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/nfs.dump.dir in system properties and HBase conf 2024-11-27T12:28:03,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/java.io.tmpdir in system properties and HBase conf 2024-11-27T12:28:03,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T12:28:03,582 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-27T12:28:03,582 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-27T12:28:04,730 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-27T12:28:04,804 INFO [Time-limited test {}] log.Log(170): Logging initialized @2587ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-27T12:28:04,875 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T12:28:04,935 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T12:28:04,964 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T12:28:04,965 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T12:28:04,966 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T12:28:04,977 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T12:28:04,986 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3c1cc886{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/hadoop.log.dir/,AVAILABLE} 2024-11-27T12:28:04,987 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a9265d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T12:28:05,161 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c411b81{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/java.io.tmpdir/jetty-localhost-32925-hadoop-hdfs-3_4_1-tests_jar-_-any-3742763377391877445/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T12:28:05,167 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@71f91499{HTTP/1.1, (http/1.1)}{localhost:32925} 2024-11-27T12:28:05,167 INFO [Time-limited test {}] server.Server(415): Started @2950ms 2024-11-27T12:28:05,720 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T12:28:05,727 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T12:28:05,728 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T12:28:05,728 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T12:28:05,729 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T12:28:05,729 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@15fab457{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/hadoop.log.dir/,AVAILABLE} 2024-11-27T12:28:05,730 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f0d3e1c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T12:28:05,827 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@53a60ac5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/java.io.tmpdir/jetty-localhost-41021-hadoop-hdfs-3_4_1-tests_jar-_-any-15931866746113723104/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T12:28:05,828 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b9d2b29{HTTP/1.1, (http/1.1)}{localhost:41021} 2024-11-27T12:28:05,828 INFO [Time-limited test {}] server.Server(415): Started @3611ms 2024-11-27T12:28:05,872 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T12:28:05,974 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T12:28:05,981 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T12:28:05,985 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T12:28:05,985 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T12:28:05,985 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T12:28:05,986 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5c155ec7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/hadoop.log.dir/,AVAILABLE} 2024-11-27T12:28:05,987 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ea2c5fa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T12:28:06,126 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@bdffcb9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/java.io.tmpdir/jetty-localhost-37967-hadoop-hdfs-3_4_1-tests_jar-_-any-15072059541501494501/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T12:28:06,127 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@55b08a17{HTTP/1.1, (http/1.1)}{localhost:37967} 2024-11-27T12:28:06,127 INFO [Time-limited test {}] server.Server(415): Started @3910ms 2024-11-27T12:28:06,131 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T12:28:06,166 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T12:28:06,170 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T12:28:06,172 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T12:28:06,172 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T12:28:06,172 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T12:28:06,174 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4870c3fc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/hadoop.log.dir/,AVAILABLE} 2024-11-27T12:28:06,175 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3cda6c9e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T12:28:06,272 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@638bb008{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/java.io.tmpdir/jetty-localhost-43751-hadoop-hdfs-3_4_1-tests_jar-_-any-1628676476664276699/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T12:28:06,273 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@394c588e{HTTP/1.1, (http/1.1)}{localhost:43751} 2024-11-27T12:28:06,273 INFO [Time-limited test {}] server.Server(415): Started @4056ms 2024-11-27T12:28:06,275 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T12:28:07,696 WARN [Thread-129 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/data/data4/current/BP-39017082-172.17.0.3-1732710484128/current, will proceed with Du for space computation calculation, 2024-11-27T12:28:07,696 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/data/data1/current/BP-39017082-172.17.0.3-1732710484128/current, will proceed with Du for space computation calculation, 2024-11-27T12:28:07,696 WARN [Thread-127 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/data/data6/current/BP-39017082-172.17.0.3-1732710484128/current, will proceed with Du for space computation calculation, 2024-11-27T12:28:07,696 WARN [Thread-126 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/data/data5/current/BP-39017082-172.17.0.3-1732710484128/current, will proceed with Du for space computation calculation, 2024-11-27T12:28:07,696 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/data/data2/current/BP-39017082-172.17.0.3-1732710484128/current, will proceed with Du for space computation calculation, 2024-11-27T12:28:07,696 WARN [Thread-128 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/data/data3/current/BP-39017082-172.17.0.3-1732710484128/current, will proceed with Du for space computation calculation, 2024-11-27T12:28:07,730 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T12:28:07,730 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T12:28:07,730 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T12:28:07,777 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2d8b89e24b2b805f with lease ID 0xc88c3707956eb822: Processing first storage report for DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf from datanode DatanodeRegistration(127.0.0.1:37469, datanodeUuid=1859aba2-a42a-4521-b053-d968c4078117, infoPort=45591, infoSecurePort=0, ipcPort=39259, storageInfo=lv=-57;cid=testClusterID;nsid=1309299118;c=1732710484128) 2024-11-27T12:28:07,778 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2d8b89e24b2b805f with lease ID 0xc88c3707956eb822: from storage DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf node DatanodeRegistration(127.0.0.1:37469, datanodeUuid=1859aba2-a42a-4521-b053-d968c4078117, infoPort=45591, infoSecurePort=0, ipcPort=39259, storageInfo=lv=-57;cid=testClusterID;nsid=1309299118;c=1732710484128), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-27T12:28:07,779 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf6808b00972074df with lease ID 0xc88c3707956eb821: Processing first storage report for DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5 from datanode DatanodeRegistration(127.0.0.1:38877, datanodeUuid=c049323e-5ead-432d-9a1c-c0835999441a, infoPort=46317, infoSecurePort=0, ipcPort=36597, storageInfo=lv=-57;cid=testClusterID;nsid=1309299118;c=1732710484128) 2024-11-27T12:28:07,779 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf6808b00972074df with lease ID 0xc88c3707956eb821: from storage DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5 node DatanodeRegistration(127.0.0.1:38877, datanodeUuid=c049323e-5ead-432d-9a1c-c0835999441a, infoPort=46317, infoSecurePort=0, ipcPort=36597, storageInfo=lv=-57;cid=testClusterID;nsid=1309299118;c=1732710484128), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T12:28:07,779 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe42980cbb8557448 with lease ID 0xc88c3707956eb823: Processing first storage report for DS-a9c334cc-4027-492e-8f93-da489a14b1de from datanode DatanodeRegistration(127.0.0.1:38365, datanodeUuid=d8a095d7-047e-41bc-87f0-5bd820a5ee93, infoPort=34427, infoSecurePort=0, ipcPort=40089, storageInfo=lv=-57;cid=testClusterID;nsid=1309299118;c=1732710484128) 2024-11-27T12:28:07,780 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe42980cbb8557448 with lease ID 0xc88c3707956eb823: from storage DS-a9c334cc-4027-492e-8f93-da489a14b1de node DatanodeRegistration(127.0.0.1:38365, datanodeUuid=d8a095d7-047e-41bc-87f0-5bd820a5ee93, infoPort=34427, infoSecurePort=0, ipcPort=40089, storageInfo=lv=-57;cid=testClusterID;nsid=1309299118;c=1732710484128), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T12:28:07,780 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2d8b89e24b2b805f with lease ID 0xc88c3707956eb822: Processing first storage report for DS-ee6418ff-a605-464b-89a6-0a25acf9a346 from datanode DatanodeRegistration(127.0.0.1:37469, datanodeUuid=1859aba2-a42a-4521-b053-d968c4078117, infoPort=45591, infoSecurePort=0, ipcPort=39259, storageInfo=lv=-57;cid=testClusterID;nsid=1309299118;c=1732710484128) 2024-11-27T12:28:07,780 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2d8b89e24b2b805f with lease ID 0xc88c3707956eb822: from storage DS-ee6418ff-a605-464b-89a6-0a25acf9a346 node DatanodeRegistration(127.0.0.1:37469, datanodeUuid=1859aba2-a42a-4521-b053-d968c4078117, infoPort=45591, infoSecurePort=0, ipcPort=39259, storageInfo=lv=-57;cid=testClusterID;nsid=1309299118;c=1732710484128), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T12:28:07,780 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf6808b00972074df with lease ID 0xc88c3707956eb821: Processing first storage report for DS-3f4401af-0199-4baa-86df-b53412f095f6 from datanode DatanodeRegistration(127.0.0.1:38877, datanodeUuid=c049323e-5ead-432d-9a1c-c0835999441a, infoPort=46317, infoSecurePort=0, ipcPort=36597, storageInfo=lv=-57;cid=testClusterID;nsid=1309299118;c=1732710484128) 2024-11-27T12:28:07,781 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf6808b00972074df with lease ID 0xc88c3707956eb821: from storage DS-3f4401af-0199-4baa-86df-b53412f095f6 node DatanodeRegistration(127.0.0.1:38877, datanodeUuid=c049323e-5ead-432d-9a1c-c0835999441a, infoPort=46317, infoSecurePort=0, ipcPort=36597, storageInfo=lv=-57;cid=testClusterID;nsid=1309299118;c=1732710484128), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T12:28:07,781 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe42980cbb8557448 with lease ID 0xc88c3707956eb823: Processing first storage report for DS-59866d9c-d5f8-40ee-b2c8-2f15f7cebd15 from datanode DatanodeRegistration(127.0.0.1:38365, datanodeUuid=d8a095d7-047e-41bc-87f0-5bd820a5ee93, infoPort=34427, infoSecurePort=0, ipcPort=40089, storageInfo=lv=-57;cid=testClusterID;nsid=1309299118;c=1732710484128) 2024-11-27T12:28:07,781 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe42980cbb8557448 with lease ID 0xc88c3707956eb823: from storage DS-59866d9c-d5f8-40ee-b2c8-2f15f7cebd15 node DatanodeRegistration(127.0.0.1:38365, datanodeUuid=d8a095d7-047e-41bc-87f0-5bd820a5ee93, infoPort=34427, infoSecurePort=0, ipcPort=40089, storageInfo=lv=-57;cid=testClusterID;nsid=1309299118;c=1732710484128), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T12:28:07,854 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5 2024-11-27T12:28:07,918 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/zookeeper_0, clientPort=58880, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T12:28:07,928 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58880 2024-11-27T12:28:07,939 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T12:28:07,943 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T12:28:08,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741825_1001 (size=7) 2024-11-27T12:28:08,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741825_1001 (size=7) 2024-11-27T12:28:08,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741825_1001 (size=7) 2024-11-27T12:28:08,561 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606 with version=8 2024-11-27T12:28:08,562 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/hbase-staging 2024-11-27T12:28:08,870 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1ba5af43fd99:0 server-side Connection retries=45 2024-11-27T12:28:08,878 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:28:08,878 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T12:28:08,884 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T12:28:08,884 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:28:08,884 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T12:28:09,013 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T12:28:09,063 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-27T12:28:09,071 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-27T12:28:09,074 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T12:28:09,096 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 76271 (auto-detected) 2024-11-27T12:28:09,097 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-27T12:28:09,114 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46311 2024-11-27T12:28:09,134 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46311 connecting to ZooKeeper ensemble=127.0.0.1:58880 2024-11-27T12:28:09,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:463110x0, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T12:28:09,288 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46311-0x1017c43d3070000 connected 2024-11-27T12:28:09,373 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T12:28:09,380 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T12:28:09,394 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:28:09,398 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606, hbase.cluster.distributed=false 2024-11-27T12:28:09,420 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T12:28:09,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46311 2024-11-27T12:28:09,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46311 2024-11-27T12:28:09,425 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46311 2024-11-27T12:28:09,425 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46311 2024-11-27T12:28:09,426 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46311 2024-11-27T12:28:09,521 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1ba5af43fd99:0 server-side Connection retries=45 2024-11-27T12:28:09,522 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:28:09,522 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T12:28:09,522 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T12:28:09,523 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:28:09,523 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T12:28:09,525 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T12:28:09,528 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T12:28:09,528 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35527 2024-11-27T12:28:09,531 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35527 connecting to ZooKeeper ensemble=127.0.0.1:58880 2024-11-27T12:28:09,532 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T12:28:09,534 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T12:28:09,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:355270x0, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T12:28:09,550 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:355270x0, quorum=127.0.0.1:58880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:28:09,550 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35527-0x1017c43d3070001 connected 2024-11-27T12:28:09,554 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T12:28:09,563 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T12:28:09,565 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35527-0x1017c43d3070001, quorum=127.0.0.1:58880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T12:28:09,571 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35527-0x1017c43d3070001, quorum=127.0.0.1:58880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T12:28:09,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35527 2024-11-27T12:28:09,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35527 2024-11-27T12:28:09,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35527 2024-11-27T12:28:09,573 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35527 2024-11-27T12:28:09,573 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35527 2024-11-27T12:28:09,588 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1ba5af43fd99:0 server-side Connection retries=45 2024-11-27T12:28:09,588 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:28:09,588 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T12:28:09,589 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T12:28:09,589 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:28:09,589 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T12:28:09,589 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T12:28:09,590 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T12:28:09,590 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41787 2024-11-27T12:28:09,592 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41787 connecting to ZooKeeper ensemble=127.0.0.1:58880 2024-11-27T12:28:09,593 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T12:28:09,596 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T12:28:09,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:417870x0, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T12:28:09,611 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41787-0x1017c43d3070002 connected 2024-11-27T12:28:09,611 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:28:09,612 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T12:28:09,613 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T12:28:09,614 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T12:28:09,617 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T12:28:09,618 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41787 2024-11-27T12:28:09,618 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41787 2024-11-27T12:28:09,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41787 2024-11-27T12:28:09,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41787 2024-11-27T12:28:09,621 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41787 2024-11-27T12:28:09,641 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1ba5af43fd99:0 server-side Connection retries=45 2024-11-27T12:28:09,641 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:28:09,641 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T12:28:09,642 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T12:28:09,642 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:28:09,642 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T12:28:09,642 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T12:28:09,642 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T12:28:09,643 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44849 2024-11-27T12:28:09,645 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44849 connecting to ZooKeeper ensemble=127.0.0.1:58880 2024-11-27T12:28:09,646 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T12:28:09,648 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T12:28:09,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:448490x0, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T12:28:09,665 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:448490x0, quorum=127.0.0.1:58880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:28:09,666 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44849-0x1017c43d3070003 connected 2024-11-27T12:28:09,666 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T12:28:09,667 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T12:28:09,668 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T12:28:09,670 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T12:28:09,671 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44849 2024-11-27T12:28:09,671 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44849 2024-11-27T12:28:09,672 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44849 2024-11-27T12:28:09,672 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44849 2024-11-27T12:28:09,673 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44849 2024-11-27T12:28:09,686 DEBUG [M:0;1ba5af43fd99:46311 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1ba5af43fd99:46311 2024-11-27T12:28:09,687 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1ba5af43fd99,46311,1732710488723 2024-11-27T12:28:09,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35527-0x1017c43d3070001, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:28:09,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:28:09,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:28:09,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:28:09,698 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1ba5af43fd99,46311,1732710488723 2024-11-27T12:28:09,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:09,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35527-0x1017c43d3070001, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T12:28:09,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T12:28:09,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T12:28:09,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35527-0x1017c43d3070001, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:09,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:09,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:09,730 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T12:28:09,731 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1ba5af43fd99,46311,1732710488723 from backup master directory 2024-11-27T12:28:09,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1ba5af43fd99,46311,1732710488723 2024-11-27T12:28:09,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:28:09,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:28:09,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35527-0x1017c43d3070001, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:28:09,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:28:09,739 WARN [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T12:28:09,739 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1ba5af43fd99,46311,1732710488723 2024-11-27T12:28:09,742 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-27T12:28:09,743 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-27T12:28:09,800 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/hbase.id] with ID: 7647a787-0323-46fc-9c11-7d9c2370932f 2024-11-27T12:28:09,800 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/.tmp/hbase.id 2024-11-27T12:28:09,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741826_1002 (size=42) 2024-11-27T12:28:09,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741826_1002 (size=42) 2024-11-27T12:28:09,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741826_1002 (size=42) 2024-11-27T12:28:09,818 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/.tmp/hbase.id]:[hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/hbase.id] 2024-11-27T12:28:09,867 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T12:28:09,872 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T12:28:09,890 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-27T12:28:09,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:09,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35527-0x1017c43d3070001, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:09,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:09,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:09,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741827_1003 (size=196) 2024-11-27T12:28:09,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741827_1003 (size=196) 2024-11-27T12:28:09,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741827_1003 (size=196) 2024-11-27T12:28:09,942 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T12:28:09,944 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T12:28:09,948 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:09,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741828_1004 (size=1189) 2024-11-27T12:28:09,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741828_1004 (size=1189) 2024-11-27T12:28:09,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741828_1004 (size=1189) 2024-11-27T12:28:09,998 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store 2024-11-27T12:28:10,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741829_1005 (size=34) 2024-11-27T12:28:10,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741829_1005 (size=34) 2024-11-27T12:28:10,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741829_1005 (size=34) 2024-11-27T12:28:10,024 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-27T12:28:10,027 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:10,028 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T12:28:10,029 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:28:10,029 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:28:10,031 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T12:28:10,031 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:28:10,031 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:28:10,032 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732710490028Disabling compacts and flushes for region at 1732710490028Disabling writes for close at 1732710490031 (+3 ms)Writing region close event to WAL at 1732710490031Closed at 1732710490031 2024-11-27T12:28:10,034 WARN [master/1ba5af43fd99:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store/.initializing 2024-11-27T12:28:10,035 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/WALs/1ba5af43fd99,46311,1732710488723 2024-11-27T12:28:10,042 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:28:10,056 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ba5af43fd99%2C46311%2C1732710488723, suffix=, logDir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/WALs/1ba5af43fd99,46311,1732710488723, archiveDir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/oldWALs, maxLogs=10 2024-11-27T12:28:10,095 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/WALs/1ba5af43fd99,46311,1732710488723/1ba5af43fd99%2C46311%2C1732710488723.1732710490062, exclude list is [], retry=0 2024-11-27T12:28:10,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:10,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:10,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:10,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:10,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-27T12:28:10,159 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/WALs/1ba5af43fd99,46311,1732710488723/1ba5af43fd99%2C46311%2C1732710488723.1732710490062 2024-11-27T12:28:10,160 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:45591:45591)] 2024-11-27T12:28:10,161 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:10,161 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:10,164 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:28:10,165 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:28:10,199 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:28:10,221 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T12:28:10,224 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:10,226 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:28:10,226 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:28:10,229 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T12:28:10,230 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:10,231 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:10,231 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:28:10,233 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T12:28:10,233 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:10,234 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:10,235 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:28:10,237 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T12:28:10,237 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:10,238 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:10,239 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:28:10,242 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:28:10,243 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:28:10,248 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:28:10,249 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:28:10,252 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T12:28:10,255 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:28:10,260 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:28:10,261 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73385348, jitterRate=0.09352689981460571}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T12:28:10,268 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732710490177Initializing all the Stores at 1732710490179 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732710490180 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710490180Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710490181 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710490181Cleaning up temporary data from old regions at 1732710490249 (+68 ms)Region opened successfully at 1732710490268 (+19 ms) 2024-11-27T12:28:10,269 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T12:28:10,299 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2866d038, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ba5af43fd99/172.17.0.3:0 2024-11-27T12:28:10,325 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T12:28:10,334 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-27T12:28:10,335 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-27T12:28:10,337 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T12:28:10,339 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-27T12:28:10,343 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-27T12:28:10,343 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T12:28:10,368 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T12:28:10,375 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T12:28:10,423 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T12:28:10,429 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T12:28:10,432 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T12:28:10,443 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T12:28:10,446 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T12:28:10,449 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T12:28:10,454 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T12:28:10,456 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T12:28:10,464 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T12:28:10,484 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T12:28:10,496 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T12:28:10,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T12:28:10,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35527-0x1017c43d3070001, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T12:28:10,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T12:28:10,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T12:28:10,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:10,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35527-0x1017c43d3070001, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:10,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:10,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:10,511 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1ba5af43fd99,46311,1732710488723, sessionid=0x1017c43d3070000, setting cluster-up flag (Was=false) 2024-11-27T12:28:10,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35527-0x1017c43d3070001, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:10,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:10,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:10,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:10,580 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T12:28:10,584 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1ba5af43fd99,46311,1732710488723 2024-11-27T12:28:10,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:10,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:10,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:10,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35527-0x1017c43d3070001, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:10,643 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T12:28:10,646 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1ba5af43fd99,46311,1732710488723 2024-11-27T12:28:10,654 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T12:28:10,676 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.HRegionServer(746): ClusterId : 7647a787-0323-46fc-9c11-7d9c2370932f 2024-11-27T12:28:10,676 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer(746): ClusterId : 7647a787-0323-46fc-9c11-7d9c2370932f 2024-11-27T12:28:10,676 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.HRegionServer(746): ClusterId : 7647a787-0323-46fc-9c11-7d9c2370932f 2024-11-27T12:28:10,679 DEBUG [RS:2;1ba5af43fd99:44849 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T12:28:10,679 DEBUG [RS:1;1ba5af43fd99:41787 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T12:28:10,679 DEBUG [RS:0;1ba5af43fd99:35527 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T12:28:10,687 DEBUG [RS:1;1ba5af43fd99:41787 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T12:28:10,687 DEBUG [RS:2;1ba5af43fd99:44849 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T12:28:10,687 DEBUG [RS:0;1ba5af43fd99:35527 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T12:28:10,687 DEBUG [RS:2;1ba5af43fd99:44849 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T12:28:10,687 DEBUG [RS:0;1ba5af43fd99:35527 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T12:28:10,687 DEBUG [RS:1;1ba5af43fd99:41787 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T12:28:10,707 DEBUG [RS:2;1ba5af43fd99:44849 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T12:28:10,707 DEBUG [RS:1;1ba5af43fd99:41787 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T12:28:10,707 DEBUG [RS:0;1ba5af43fd99:35527 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T12:28:10,708 DEBUG [RS:0;1ba5af43fd99:35527 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d94005, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ba5af43fd99/172.17.0.3:0 2024-11-27T12:28:10,708 DEBUG [RS:1;1ba5af43fd99:41787 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f8bd0a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ba5af43fd99/172.17.0.3:0 2024-11-27T12:28:10,708 DEBUG [RS:2;1ba5af43fd99:44849 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71950335, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ba5af43fd99/172.17.0.3:0 2024-11-27T12:28:10,721 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T12:28:10,722 DEBUG [RS:0;1ba5af43fd99:35527 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1ba5af43fd99:35527 2024-11-27T12:28:10,723 DEBUG [RS:1;1ba5af43fd99:41787 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;1ba5af43fd99:41787 2024-11-27T12:28:10,725 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T12:28:10,725 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T12:28:10,726 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T12:28:10,726 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T12:28:10,726 DEBUG [RS:1;1ba5af43fd99:41787 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T12:28:10,726 DEBUG [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T12:28:10,728 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.HRegionServer(2659): reportForDuty to master=1ba5af43fd99,46311,1732710488723 with port=41787, startcode=1732710489587 2024-11-27T12:28:10,728 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer(2659): reportForDuty to master=1ba5af43fd99,46311,1732710488723 with port=35527, startcode=1732710489490 2024-11-27T12:28:10,729 DEBUG [RS:2;1ba5af43fd99:44849 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;1ba5af43fd99:44849 2024-11-27T12:28:10,729 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T12:28:10,729 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T12:28:10,729 DEBUG [RS:2;1ba5af43fd99:44849 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T12:28:10,730 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.HRegionServer(2659): reportForDuty to master=1ba5af43fd99,46311,1732710488723 with port=44849, startcode=1732710489640 2024-11-27T12:28:10,730 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T12:28:10,737 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T12:28:10,738 DEBUG [RS:1;1ba5af43fd99:41787 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T12:28:10,738 DEBUG [RS:2;1ba5af43fd99:44849 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T12:28:10,738 DEBUG [RS:0;1ba5af43fd99:35527 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T12:28:10,742 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1ba5af43fd99,46311,1732710488723 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T12:28:10,749 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1ba5af43fd99:0, corePoolSize=5, maxPoolSize=5 2024-11-27T12:28:10,749 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1ba5af43fd99:0, corePoolSize=5, maxPoolSize=5 2024-11-27T12:28:10,749 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1ba5af43fd99:0, corePoolSize=5, maxPoolSize=5 2024-11-27T12:28:10,749 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1ba5af43fd99:0, corePoolSize=5, maxPoolSize=5 2024-11-27T12:28:10,750 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1ba5af43fd99:0, corePoolSize=10, maxPoolSize=10 2024-11-27T12:28:10,750 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:10,750 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1ba5af43fd99:0, corePoolSize=2, maxPoolSize=2 2024-11-27T12:28:10,750 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:10,762 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732710520762 2024-11-27T12:28:10,764 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T12:28:10,764 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T12:28:10,765 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T12:28:10,765 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T12:28:10,769 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T12:28:10,769 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T12:28:10,769 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T12:28:10,769 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T12:28:10,770 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:10,770 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T12:28:10,773 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41637, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T12:28:10,773 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44227, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T12:28:10,773 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55663, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T12:28:10,770 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:10,775 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T12:28:10,777 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T12:28:10,777 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T12:28:10,779 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T12:28:10,779 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T12:28:10,782 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46311 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:28:10,784 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1ba5af43fd99:0:becomeActiveMaster-HFileCleaner.large.0-1732710490780,5,FailOnTimeoutGroup] 2024-11-27T12:28:10,785 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1ba5af43fd99:0:becomeActiveMaster-HFileCleaner.small.0-1732710490784,5,FailOnTimeoutGroup] 2024-11-27T12:28:10,785 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:10,785 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T12:28:10,805 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46311 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:28:10,805 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:10,806 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:10,807 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46311 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1ba5af43fd99,44849,1732710489640 2024-11-27T12:28:10,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741831_1007 (size=1321) 2024-11-27T12:28:10,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741831_1007 (size=1321) 2024-11-27T12:28:10,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741831_1007 (size=1321) 2024-11-27T12:28:10,810 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46311 {}] master.ServerManager(517): Registering regionserver=1ba5af43fd99,44849,1732710489640 2024-11-27T12:28:10,812 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T12:28:10,812 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606 2024-11-27T12:28:10,819 DEBUG [RS:2;1ba5af43fd99:44849 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606 2024-11-27T12:28:10,820 DEBUG [RS:2;1ba5af43fd99:44849 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38485 2024-11-27T12:28:10,820 DEBUG [RS:2;1ba5af43fd99:44849 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T12:28:10,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741832_1008 (size=32) 2024-11-27T12:28:10,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741832_1008 (size=32) 2024-11-27T12:28:10,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741832_1008 (size=32) 2024-11-27T12:28:10,829 DEBUG [RS:1;1ba5af43fd99:41787 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-27T12:28:10,829 DEBUG [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-27T12:28:10,829 WARN [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-27T12:28:10,829 WARN [RS:1;1ba5af43fd99:41787 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-27T12:28:10,830 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:10,832 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T12:28:10,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T12:28:10,834 DEBUG [RS:2;1ba5af43fd99:44849 {}] zookeeper.ZKUtil(111): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1ba5af43fd99,44849,1732710489640 2024-11-27T12:28:10,834 WARN [RS:2;1ba5af43fd99:44849 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T12:28:10,834 INFO [RS:2;1ba5af43fd99:44849 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:10,834 DEBUG [RS:2;1ba5af43fd99:44849 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,44849,1732710489640 2024-11-27T12:28:10,836 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T12:28:10,836 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1ba5af43fd99,44849,1732710489640] 2024-11-27T12:28:10,836 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:10,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:28:10,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T12:28:10,840 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T12:28:10,840 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:10,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:28:10,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T12:28:10,844 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T12:28:10,845 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:10,846 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:28:10,846 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T12:28:10,849 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T12:28:10,849 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:10,850 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:28:10,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T12:28:10,852 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740 2024-11-27T12:28:10,853 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740 2024-11-27T12:28:10,856 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T12:28:10,856 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T12:28:10,857 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T12:28:10,859 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T12:28:10,862 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T12:28:10,863 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:28:10,864 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60575971, jitterRate=-0.097347691655159}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T12:28:10,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732710490830Initializing all the Stores at 1732710490832 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732710490832Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732710490832Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710490832Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732710490832Cleaning up temporary data from old regions at 1732710490856 (+24 ms)Region opened successfully at 1732710490868 (+12 ms) 2024-11-27T12:28:10,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T12:28:10,868 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T12:28:10,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T12:28:10,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T12:28:10,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T12:28:10,870 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T12:28:10,870 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732710490868Disabling compacts and flushes for region at 1732710490868Disabling writes for close at 1732710490868Writing region close event to WAL at 1732710490869 (+1 ms)Closed at 1732710490870 (+1 ms) 2024-11-27T12:28:10,874 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T12:28:10,874 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T12:28:10,878 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T12:28:10,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T12:28:10,883 INFO [RS:2;1ba5af43fd99:44849 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T12:28:10,883 INFO [RS:2;1ba5af43fd99:44849 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:10,884 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T12:28:10,889 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T12:28:10,890 INFO [RS:2;1ba5af43fd99:44849 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T12:28:10,892 INFO [RS:2;1ba5af43fd99:44849 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:10,892 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T12:28:10,893 DEBUG [RS:2;1ba5af43fd99:44849 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:10,893 DEBUG [RS:2;1ba5af43fd99:44849 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:10,893 DEBUG [RS:2;1ba5af43fd99:44849 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:10,893 DEBUG [RS:2;1ba5af43fd99:44849 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:10,893 DEBUG [RS:2;1ba5af43fd99:44849 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:10,894 DEBUG [RS:2;1ba5af43fd99:44849 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0, corePoolSize=2, maxPoolSize=2 2024-11-27T12:28:10,894 DEBUG [RS:2;1ba5af43fd99:44849 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:10,894 DEBUG [RS:2;1ba5af43fd99:44849 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:10,894 DEBUG [RS:2;1ba5af43fd99:44849 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:10,894 DEBUG [RS:2;1ba5af43fd99:44849 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:10,894 DEBUG [RS:2;1ba5af43fd99:44849 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:10,894 DEBUG [RS:2;1ba5af43fd99:44849 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:10,894 DEBUG [RS:2;1ba5af43fd99:44849 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1ba5af43fd99:0, corePoolSize=3, maxPoolSize=3 2024-11-27T12:28:10,895 DEBUG [RS:2;1ba5af43fd99:44849 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0, corePoolSize=3, maxPoolSize=3 2024-11-27T12:28:10,895 INFO [RS:2;1ba5af43fd99:44849 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:10,896 INFO [RS:2;1ba5af43fd99:44849 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:10,896 INFO [RS:2;1ba5af43fd99:44849 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:10,896 INFO [RS:2;1ba5af43fd99:44849 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:10,896 INFO [RS:2;1ba5af43fd99:44849 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:10,896 INFO [RS:2;1ba5af43fd99:44849 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,44849,1732710489640-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T12:28:10,915 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T12:28:10,916 INFO [RS:2;1ba5af43fd99:44849 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,44849,1732710489640-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:10,916 INFO [RS:2;1ba5af43fd99:44849 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:10,917 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.Replication(171): 1ba5af43fd99,44849,1732710489640 started 2024-11-27T12:28:10,931 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.HRegionServer(2659): reportForDuty to master=1ba5af43fd99,46311,1732710488723 with port=41787, startcode=1732710489587 2024-11-27T12:28:10,931 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer(2659): reportForDuty to master=1ba5af43fd99,46311,1732710488723 with port=35527, startcode=1732710489490 2024-11-27T12:28:10,932 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46311 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1ba5af43fd99,35527,1732710489490 2024-11-27T12:28:10,932 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46311 {}] master.ServerManager(517): Registering regionserver=1ba5af43fd99,35527,1732710489490 2024-11-27T12:28:10,933 INFO [RS:2;1ba5af43fd99:44849 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:10,933 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.HRegionServer(1482): Serving as 1ba5af43fd99,44849,1732710489640, RpcServer on 1ba5af43fd99/172.17.0.3:44849, sessionid=0x1017c43d3070003 2024-11-27T12:28:10,934 DEBUG [RS:2;1ba5af43fd99:44849 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T12:28:10,934 DEBUG [RS:2;1ba5af43fd99:44849 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1ba5af43fd99,44849,1732710489640 2024-11-27T12:28:10,934 DEBUG [RS:2;1ba5af43fd99:44849 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ba5af43fd99,44849,1732710489640' 2024-11-27T12:28:10,934 DEBUG [RS:2;1ba5af43fd99:44849 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T12:28:10,935 DEBUG [RS:2;1ba5af43fd99:44849 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T12:28:10,936 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46311 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1ba5af43fd99,41787,1732710489587 2024-11-27T12:28:10,936 DEBUG [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606 2024-11-27T12:28:10,936 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46311 {}] master.ServerManager(517): Registering regionserver=1ba5af43fd99,41787,1732710489587 2024-11-27T12:28:10,936 DEBUG [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38485 2024-11-27T12:28:10,936 DEBUG [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T12:28:10,936 DEBUG [RS:2;1ba5af43fd99:44849 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T12:28:10,936 DEBUG [RS:2;1ba5af43fd99:44849 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T12:28:10,936 DEBUG [RS:2;1ba5af43fd99:44849 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1ba5af43fd99,44849,1732710489640 2024-11-27T12:28:10,936 DEBUG [RS:2;1ba5af43fd99:44849 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ba5af43fd99,44849,1732710489640' 2024-11-27T12:28:10,936 DEBUG [RS:2;1ba5af43fd99:44849 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T12:28:10,939 DEBUG [RS:1;1ba5af43fd99:41787 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606 2024-11-27T12:28:10,939 DEBUG [RS:1;1ba5af43fd99:41787 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38485 2024-11-27T12:28:10,939 DEBUG [RS:1;1ba5af43fd99:41787 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T12:28:10,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T12:28:10,980 DEBUG [RS:2;1ba5af43fd99:44849 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T12:28:11,001 DEBUG [RS:0;1ba5af43fd99:35527 {}] zookeeper.ZKUtil(111): regionserver:35527-0x1017c43d3070001, quorum=127.0.0.1:58880, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1ba5af43fd99,35527,1732710489490 2024-11-27T12:28:11,001 DEBUG [RS:2;1ba5af43fd99:44849 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T12:28:11,002 WARN [RS:0;1ba5af43fd99:35527 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T12:28:11,002 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1ba5af43fd99,35527,1732710489490] 2024-11-27T12:28:11,002 INFO [RS:2;1ba5af43fd99:44849 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T12:28:11,002 INFO [RS:0;1ba5af43fd99:35527 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:11,002 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1ba5af43fd99,41787,1732710489587] 2024-11-27T12:28:11,002 INFO [RS:2;1ba5af43fd99:44849 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T12:28:11,002 DEBUG [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490 2024-11-27T12:28:11,002 DEBUG [RS:1;1ba5af43fd99:41787 {}] zookeeper.ZKUtil(111): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1ba5af43fd99,41787,1732710489587 2024-11-27T12:28:11,002 WARN [RS:1;1ba5af43fd99:41787 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T12:28:11,002 INFO [RS:1;1ba5af43fd99:41787 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:11,002 DEBUG [RS:1;1ba5af43fd99:41787 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,41787,1732710489587 2024-11-27T12:28:11,008 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T12:28:11,008 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T12:28:11,011 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T12:28:11,011 INFO [RS:0;1ba5af43fd99:35527 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T12:28:11,011 INFO [RS:0;1ba5af43fd99:35527 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,012 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T12:28:11,014 INFO [RS:0;1ba5af43fd99:35527 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T12:28:11,014 INFO [RS:0;1ba5af43fd99:35527 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,014 DEBUG [RS:0;1ba5af43fd99:35527 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,015 DEBUG [RS:0;1ba5af43fd99:35527 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,015 DEBUG [RS:0;1ba5af43fd99:35527 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,015 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T12:28:11,015 DEBUG [RS:0;1ba5af43fd99:35527 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,015 DEBUG [RS:0;1ba5af43fd99:35527 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,015 DEBUG [RS:0;1ba5af43fd99:35527 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0, corePoolSize=2, maxPoolSize=2 2024-11-27T12:28:11,015 INFO [RS:1;1ba5af43fd99:41787 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T12:28:11,015 INFO [RS:1;1ba5af43fd99:41787 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,015 DEBUG [RS:0;1ba5af43fd99:35527 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,016 DEBUG [RS:0;1ba5af43fd99:35527 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,016 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T12:28:11,016 DEBUG [RS:0;1ba5af43fd99:35527 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,016 DEBUG [RS:0;1ba5af43fd99:35527 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,016 DEBUG [RS:0;1ba5af43fd99:35527 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,016 DEBUG [RS:0;1ba5af43fd99:35527 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,016 DEBUG [RS:0;1ba5af43fd99:35527 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1ba5af43fd99:0, corePoolSize=3, maxPoolSize=3 2024-11-27T12:28:11,016 DEBUG [RS:0;1ba5af43fd99:35527 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0, corePoolSize=3, maxPoolSize=3 2024-11-27T12:28:11,017 INFO [RS:0;1ba5af43fd99:35527 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,017 INFO [RS:0;1ba5af43fd99:35527 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,017 INFO [RS:1;1ba5af43fd99:41787 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T12:28:11,017 INFO [RS:0;1ba5af43fd99:35527 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,017 INFO [RS:0;1ba5af43fd99:35527 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,017 INFO [RS:0;1ba5af43fd99:35527 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,017 INFO [RS:1;1ba5af43fd99:41787 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,017 INFO [RS:0;1ba5af43fd99:35527 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,35527,1732710489490-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T12:28:11,018 DEBUG [RS:1;1ba5af43fd99:41787 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,018 DEBUG [RS:1;1ba5af43fd99:41787 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,018 DEBUG [RS:1;1ba5af43fd99:41787 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,018 DEBUG [RS:1;1ba5af43fd99:41787 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,018 DEBUG [RS:1;1ba5af43fd99:41787 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,018 DEBUG [RS:1;1ba5af43fd99:41787 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0, corePoolSize=2, maxPoolSize=2 2024-11-27T12:28:11,019 DEBUG [RS:1;1ba5af43fd99:41787 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,019 DEBUG [RS:1;1ba5af43fd99:41787 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,019 DEBUG [RS:1;1ba5af43fd99:41787 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,019 DEBUG [RS:1;1ba5af43fd99:41787 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,019 DEBUG [RS:1;1ba5af43fd99:41787 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,019 DEBUG [RS:1;1ba5af43fd99:41787 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:28:11,020 DEBUG [RS:1;1ba5af43fd99:41787 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1ba5af43fd99:0, corePoolSize=3, maxPoolSize=3 2024-11-27T12:28:11,020 DEBUG [RS:1;1ba5af43fd99:41787 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0, corePoolSize=3, maxPoolSize=3 2024-11-27T12:28:11,028 INFO [RS:1;1ba5af43fd99:41787 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,028 INFO [RS:1;1ba5af43fd99:41787 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,028 INFO [RS:1;1ba5af43fd99:41787 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,028 INFO [RS:1;1ba5af43fd99:41787 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,028 INFO [RS:1;1ba5af43fd99:41787 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,029 INFO [RS:1;1ba5af43fd99:41787 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,41787,1732710489587-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T12:28:11,032 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T12:28:11,032 INFO [RS:0;1ba5af43fd99:35527 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,35527,1732710489490-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,033 INFO [RS:0;1ba5af43fd99:35527 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,033 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.Replication(171): 1ba5af43fd99,35527,1732710489490 started 2024-11-27T12:28:11,043 WARN [1ba5af43fd99:46311 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T12:28:11,044 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T12:28:11,044 INFO [RS:1;1ba5af43fd99:41787 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,41787,1732710489587-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,045 INFO [RS:1;1ba5af43fd99:41787 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,045 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.Replication(171): 1ba5af43fd99,41787,1732710489587 started 2024-11-27T12:28:11,048 INFO [RS:0;1ba5af43fd99:35527 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,048 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer(1482): Serving as 1ba5af43fd99,35527,1732710489490, RpcServer on 1ba5af43fd99/172.17.0.3:35527, sessionid=0x1017c43d3070001 2024-11-27T12:28:11,048 DEBUG [RS:0;1ba5af43fd99:35527 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T12:28:11,048 DEBUG [RS:0;1ba5af43fd99:35527 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1ba5af43fd99,35527,1732710489490 2024-11-27T12:28:11,048 DEBUG [RS:0;1ba5af43fd99:35527 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ba5af43fd99,35527,1732710489490' 2024-11-27T12:28:11,049 DEBUG [RS:0;1ba5af43fd99:35527 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T12:28:11,049 DEBUG [RS:0;1ba5af43fd99:35527 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T12:28:11,050 DEBUG [RS:0;1ba5af43fd99:35527 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T12:28:11,050 DEBUG [RS:0;1ba5af43fd99:35527 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T12:28:11,050 DEBUG [RS:0;1ba5af43fd99:35527 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1ba5af43fd99,35527,1732710489490 2024-11-27T12:28:11,050 DEBUG [RS:0;1ba5af43fd99:35527 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ba5af43fd99,35527,1732710489490' 2024-11-27T12:28:11,050 DEBUG [RS:0;1ba5af43fd99:35527 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T12:28:11,051 DEBUG [RS:0;1ba5af43fd99:35527 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T12:28:11,051 DEBUG [RS:0;1ba5af43fd99:35527 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T12:28:11,051 INFO [RS:0;1ba5af43fd99:35527 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T12:28:11,051 INFO [RS:0;1ba5af43fd99:35527 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T12:28:11,059 INFO [RS:1;1ba5af43fd99:41787 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,059 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.HRegionServer(1482): Serving as 1ba5af43fd99,41787,1732710489587, RpcServer on 1ba5af43fd99/172.17.0.3:41787, sessionid=0x1017c43d3070002 2024-11-27T12:28:11,059 DEBUG [RS:1;1ba5af43fd99:41787 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T12:28:11,059 DEBUG [RS:1;1ba5af43fd99:41787 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1ba5af43fd99,41787,1732710489587 2024-11-27T12:28:11,059 DEBUG [RS:1;1ba5af43fd99:41787 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ba5af43fd99,41787,1732710489587' 2024-11-27T12:28:11,059 DEBUG [RS:1;1ba5af43fd99:41787 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T12:28:11,060 DEBUG [RS:1;1ba5af43fd99:41787 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T12:28:11,061 DEBUG [RS:1;1ba5af43fd99:41787 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T12:28:11,061 DEBUG [RS:1;1ba5af43fd99:41787 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T12:28:11,061 DEBUG [RS:1;1ba5af43fd99:41787 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1ba5af43fd99,41787,1732710489587 2024-11-27T12:28:11,061 DEBUG [RS:1;1ba5af43fd99:41787 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ba5af43fd99,41787,1732710489587' 2024-11-27T12:28:11,061 DEBUG [RS:1;1ba5af43fd99:41787 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T12:28:11,062 DEBUG [RS:1;1ba5af43fd99:41787 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T12:28:11,063 DEBUG [RS:1;1ba5af43fd99:41787 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T12:28:11,063 INFO [RS:1;1ba5af43fd99:41787 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T12:28:11,063 INFO [RS:1;1ba5af43fd99:41787 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T12:28:11,110 INFO [RS:2;1ba5af43fd99:44849 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:28:11,113 INFO [RS:2;1ba5af43fd99:44849 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ba5af43fd99%2C44849%2C1732710489640, suffix=, logDir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,44849,1732710489640, archiveDir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/oldWALs, maxLogs=32 2024-11-27T12:28:11,131 DEBUG [RS:2;1ba5af43fd99:44849 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,44849,1732710489640/1ba5af43fd99%2C44849%2C1732710489640.1732710491115, exclude list is [], retry=0 2024-11-27T12:28:11,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:11,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:11,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:11,140 INFO [RS:2;1ba5af43fd99:44849 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,44849,1732710489640/1ba5af43fd99%2C44849%2C1732710489640.1732710491115 2024-11-27T12:28:11,140 DEBUG [RS:2;1ba5af43fd99:44849 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:46317:46317)] 2024-11-27T12:28:11,152 INFO [RS:0;1ba5af43fd99:35527 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:28:11,155 INFO [RS:0;1ba5af43fd99:35527 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ba5af43fd99%2C35527%2C1732710489490, suffix=, logDir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490, archiveDir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/oldWALs, maxLogs=32 2024-11-27T12:28:11,164 INFO [RS:1;1ba5af43fd99:41787 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:28:11,167 INFO [RS:1;1ba5af43fd99:41787 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ba5af43fd99%2C41787%2C1732710489587, suffix=, logDir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,41787,1732710489587, archiveDir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/oldWALs, maxLogs=32 2024-11-27T12:28:11,172 DEBUG [RS:0;1ba5af43fd99:35527 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490/1ba5af43fd99%2C35527%2C1732710489490.1732710491157, exclude list is [], retry=0 2024-11-27T12:28:11,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:11,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:11,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:11,183 DEBUG [RS:1;1ba5af43fd99:41787 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,41787,1732710489587/1ba5af43fd99%2C41787%2C1732710489587.1732710491169, exclude list is [], retry=0 2024-11-27T12:28:11,185 INFO [RS:0;1ba5af43fd99:35527 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490/1ba5af43fd99%2C35527%2C1732710489490.1732710491157 2024-11-27T12:28:11,186 DEBUG [RS:0;1ba5af43fd99:35527 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:45591:45591)] 2024-11-27T12:28:11,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:11,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:11,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:11,196 INFO [RS:1;1ba5af43fd99:41787 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,41787,1732710489587/1ba5af43fd99%2C41787%2C1732710489587.1732710491169 2024-11-27T12:28:11,197 DEBUG [RS:1;1ba5af43fd99:41787 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:45591:45591)] 2024-11-27T12:28:11,297 DEBUG [1ba5af43fd99:46311 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-27T12:28:11,307 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(204): Hosts are {1ba5af43fd99=0} racks are {/default-rack=0} 2024-11-27T12:28:11,313 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T12:28:11,313 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T12:28:11,313 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-27T12:28:11,313 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T12:28:11,313 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T12:28:11,313 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-27T12:28:11,314 INFO [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T12:28:11,314 INFO [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T12:28:11,314 INFO [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-27T12:28:11,314 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T12:28:11,320 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1ba5af43fd99,35527,1732710489490 2024-11-27T12:28:11,327 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1ba5af43fd99,35527,1732710489490, state=OPENING 2024-11-27T12:28:11,338 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T12:28:11,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:11,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35527-0x1017c43d3070001, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:11,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:11,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:11,350 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:28:11,350 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:28:11,350 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:28:11,350 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:28:11,352 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T12:28:11,355 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1ba5af43fd99,35527,1732710489490}] 2024-11-27T12:28:11,537 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T12:28:11,539 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40881, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T12:28:11,551 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T12:28:11,552 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:11,552 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T12:28:11,555 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ba5af43fd99%2C35527%2C1732710489490.meta, suffix=.meta, logDir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490, archiveDir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/oldWALs, maxLogs=32 2024-11-27T12:28:11,570 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490/1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta, exclude list is [], retry=0 2024-11-27T12:28:11,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:11,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:11,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:11,578 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490/1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta 2024-11-27T12:28:11,579 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:45591:45591)] 2024-11-27T12:28:11,579 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:11,581 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T12:28:11,584 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T12:28:11,588 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T12:28:11,592 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T12:28:11,593 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:11,593 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T12:28:11,593 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T12:28:11,597 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T12:28:11,598 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T12:28:11,598 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:11,599 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:28:11,600 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T12:28:11,601 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T12:28:11,601 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:11,602 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:28:11,602 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T12:28:11,604 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T12:28:11,604 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:11,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:28:11,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T12:28:11,607 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T12:28:11,607 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:11,608 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:28:11,609 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T12:28:11,611 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740 2024-11-27T12:28:11,614 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740 2024-11-27T12:28:11,617 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T12:28:11,617 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T12:28:11,617 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T12:28:11,620 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T12:28:11,622 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69335058, jitterRate=0.03317287564277649}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T12:28:11,622 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T12:28:11,623 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732710491594Writing region info on filesystem at 1732710491594Initializing all the Stores at 1732710491596 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732710491596Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732710491596Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710491596Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732710491596Cleaning up temporary data from old regions at 1732710491617 (+21 ms)Running coprocessor post-open hooks at 1732710491622 (+5 ms)Region opened successfully at 1732710491623 (+1 ms) 2024-11-27T12:28:11,629 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732710491527 2024-11-27T12:28:11,640 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T12:28:11,640 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T12:28:11,642 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1ba5af43fd99,35527,1732710489490 2024-11-27T12:28:11,643 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1ba5af43fd99,35527,1732710489490, state=OPEN 2024-11-27T12:28:11,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:28:11,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35527-0x1017c43d3070001, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:28:11,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:28:11,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:28:11,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:28:11,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:28:11,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:28:11,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:28:11,655 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1ba5af43fd99,35527,1732710489490 2024-11-27T12:28:11,660 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T12:28:11,660 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1ba5af43fd99,35527,1732710489490 in 300 msec 2024-11-27T12:28:11,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T12:28:11,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 783 msec 2024-11-27T12:28:11,669 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T12:28:11,669 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T12:28:11,688 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:28:11,689 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1] 2024-11-27T12:28:11,706 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:28:11,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38263, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:28:11,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0460 sec 2024-11-27T12:28:11,729 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732710491729, completionTime=-1 2024-11-27T12:28:11,732 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-27T12:28:11,732 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T12:28:11,754 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-27T12:28:11,754 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732710551754 2024-11-27T12:28:11,754 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732710611754 2024-11-27T12:28:11,754 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-11-27T12:28:11,756 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-27T12:28:11,762 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,46311,1732710488723-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,762 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,46311,1732710488723-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,763 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,46311,1732710488723-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,764 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1ba5af43fd99:46311, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,764 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,765 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,770 DEBUG [master/1ba5af43fd99:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T12:28:11,791 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.051sec 2024-11-27T12:28:11,793 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T12:28:11,794 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T12:28:11,794 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T12:28:11,795 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T12:28:11,795 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T12:28:11,795 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,46311,1732710488723-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T12:28:11,796 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,46311,1732710488723-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T12:28:11,799 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T12:28:11,800 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T12:28:11,801 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,46311,1732710488723-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:11,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66c41336, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:28:11,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,46311,-1 for getting cluster id 2024-11-27T12:28:11,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:28:11,907 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7647a787-0323-46fc-9c11-7d9c2370932f' 2024-11-27T12:28:11,910 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:28:11,910 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7647a787-0323-46fc-9c11-7d9c2370932f" 2024-11-27T12:28:11,910 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a2940d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:28:11,911 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,46311,-1] 2024-11-27T12:28:11,914 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:28:11,917 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:28:11,917 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:28:11,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6b545, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:28:11,920 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:28:11,927 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1] 2024-11-27T12:28:11,928 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:28:11,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:28:11,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1ba5af43fd99,46311,1732710488723 2024-11-27T12:28:11,951 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:38485/hbase 2024-11-27T12:28:11,967 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=368, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=343, ProcessCount=11, AvailableMemoryMB=9252 2024-11-27T12:28:11,987 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:11,990 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:11,992 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:28:11,997 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-38224853, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/hregion-38224853, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:12,012 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-38224853/hregion-38224853.1732710491999, exclude list is [], retry=0 2024-11-27T12:28:12,017 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:12,018 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:12,018 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:12,021 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-38224853/hregion-38224853.1732710491999 2024-11-27T12:28:12,023 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:34427:34427)] 2024-11-27T12:28:12,023 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6fefb17d13882c8099899efd570f7e45, NAME => 'testReplayEditsWrittenIntoWAL,,1732710491987.6fefb17d13882c8099899efd570f7e45.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38485/hbase 2024-11-27T12:28:12,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741838_1014 (size=64) 2024-11-27T12:28:12,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741838_1014 (size=64) 2024-11-27T12:28:12,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741838_1014 (size=64) 2024-11-27T12:28:12,037 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732710491987.6fefb17d13882c8099899efd570f7e45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:12,039 INFO [StoreOpener-6fefb17d13882c8099899efd570f7e45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6fefb17d13882c8099899efd570f7e45 2024-11-27T12:28:12,042 INFO [StoreOpener-6fefb17d13882c8099899efd570f7e45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fefb17d13882c8099899efd570f7e45 columnFamilyName a 2024-11-27T12:28:12,042 DEBUG [StoreOpener-6fefb17d13882c8099899efd570f7e45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:12,043 INFO [StoreOpener-6fefb17d13882c8099899efd570f7e45-1 {}] regionserver.HStore(327): Store=6fefb17d13882c8099899efd570f7e45/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:12,043 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6fefb17d13882c8099899efd570f7e45 2024-11-27T12:28:12,044 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45 2024-11-27T12:28:12,045 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45 2024-11-27T12:28:12,045 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6fefb17d13882c8099899efd570f7e45 2024-11-27T12:28:12,046 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6fefb17d13882c8099899efd570f7e45 2024-11-27T12:28:12,048 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6fefb17d13882c8099899efd570f7e45 2024-11-27T12:28:12,051 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:28:12,052 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6fefb17d13882c8099899efd570f7e45; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60440193, jitterRate=-0.09937094151973724}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T12:28:12,053 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6fefb17d13882c8099899efd570f7e45: Writing region info on filesystem at 1732710492038Initializing all the Stores at 1732710492039 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710492039Cleaning up temporary data from old regions at 1732710492046 (+7 ms)Region opened successfully at 1732710492053 (+7 ms) 2024-11-27T12:28:12,053 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6fefb17d13882c8099899efd570f7e45, disabling compactions & flushes 2024-11-27T12:28:12,054 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732710491987.6fefb17d13882c8099899efd570f7e45. 2024-11-27T12:28:12,054 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732710491987.6fefb17d13882c8099899efd570f7e45. 2024-11-27T12:28:12,054 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732710491987.6fefb17d13882c8099899efd570f7e45. after waiting 0 ms 2024-11-27T12:28:12,054 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732710491987.6fefb17d13882c8099899efd570f7e45. 2024-11-27T12:28:12,054 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732710491987.6fefb17d13882c8099899efd570f7e45. 2024-11-27T12:28:12,054 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6fefb17d13882c8099899efd570f7e45: Waiting for close lock at 1732710492053Disabling compacts and flushes for region at 1732710492053Disabling writes for close at 1732710492054 (+1 ms)Writing region close event to WAL at 1732710492054Closed at 1732710492054 2024-11-27T12:28:12,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741837_1013 (size=93) 2024-11-27T12:28:12,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741837_1013 (size=93) 2024-11-27T12:28:12,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741837_1013 (size=93) 2024-11-27T12:28:12,065 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T12:28:12,066 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-38224853:(num 1732710491999) 2024-11-27T12:28:12,068 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-27T12:28:12,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741839_1015 (size=369) 2024-11-27T12:28:12,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741839_1015 (size=369) 2024-11-27T12:28:12,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741839_1015 (size=369) 2024-11-27T12:28:12,086 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-27T12:28:12,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741840_1016 (size=231) 2024-11-27T12:28:12,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741840_1016 (size=231) 2024-11-27T12:28:12,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741840_1016 (size=231) 2024-11-27T12:28:12,119 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-1, size=369 (369bytes) 2024-11-27T12:28:12,120 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-27T12:28:12,120 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-27T12:28:12,120 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-1 2024-11-27T12:28:12,126 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-1 after 4ms 2024-11-27T12:28:12,132 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-1 took 14ms 2024-11-27T12:28:12,136 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-1 so closing down 2024-11-27T12:28:12,140 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-27T12:28:12,142 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/recovered.edits/0000000000000000001-wal-1.temp 2024-11-27T12:28:12,142 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T12:28:12,143 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T12:28:12,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741841_1017 (size=369) 2024-11-27T12:28:12,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741841_1017 (size=369) 2024-11-27T12:28:12,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741841_1017 (size=369) 2024-11-27T12:28:12,172 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-27T12:28:12,175 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/recovered.edits/0000000000000000002 2024-11-27T12:28:12,180 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-27T12:28:12,180 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-1, journal: Splitting hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-1, size=369 (369bytes) at 1732710492120Finishing writing output for hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-1 so closing down at 1732710492136 (+16 ms)Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/recovered.edits/0000000000000000001-wal-1.temp at 1732710492142 (+6 ms)3 split writer threads finished at 1732710492143 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732710492172 (+29 ms)Rename recovered edits hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/recovered.edits/0000000000000000002 at 1732710492175 (+3 ms)Processed 2 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1732710492180 (+5 ms) 2024-11-27T12:28:12,197 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-2, size=231 (231bytes) 2024-11-27T12:28:12,197 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-2 2024-11-27T12:28:12,198 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-2 after 1ms 2024-11-27T12:28:12,206 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-2 took 9ms 2024-11-27T12:28:12,209 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-2 so closing down 2024-11-27T12:28:12,209 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T12:28:12,212 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-27T12:28:12,213 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/recovered.edits/0000000000000000002-wal-2.temp 2024-11-27T12:28:12,214 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T12:28:12,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741842_1018 (size=231) 2024-11-27T12:28:12,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741842_1018 (size=231) 2024-11-27T12:28:12,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741842_1018 (size=231) 2024-11-27T12:28:12,224 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-27T12:28:12,229 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-11-27T12:28:12,231 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-27T12:28:12,231 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-2, journal: Splitting hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-2, size=231 (231bytes) at 1732710492197Finishing writing output for hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-2 so closing down at 1732710492209 (+12 ms)Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/recovered.edits/0000000000000000002-wal-2.temp at 1732710492213 (+4 ms)3 split writer threads finished at 1732710492214 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732710492224 (+10 ms)Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1732710492231 (+7 ms) 2024-11-27T12:28:12,231 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:12,234 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:12,252 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal.1732710492235, exclude list is [], retry=0 2024-11-27T12:28:12,256 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:12,257 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:12,257 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:12,259 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal.1732710492235 2024-11-27T12:28:12,260 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:46317:46317)] 2024-11-27T12:28:12,260 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6fefb17d13882c8099899efd570f7e45, NAME => 'testReplayEditsWrittenIntoWAL,,1732710491987.6fefb17d13882c8099899efd570f7e45.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:12,260 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732710491987.6fefb17d13882c8099899efd570f7e45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:12,261 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6fefb17d13882c8099899efd570f7e45 2024-11-27T12:28:12,261 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6fefb17d13882c8099899efd570f7e45 2024-11-27T12:28:12,263 INFO [StoreOpener-6fefb17d13882c8099899efd570f7e45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6fefb17d13882c8099899efd570f7e45 2024-11-27T12:28:12,265 INFO [StoreOpener-6fefb17d13882c8099899efd570f7e45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fefb17d13882c8099899efd570f7e45 columnFamilyName a 2024-11-27T12:28:12,265 DEBUG [StoreOpener-6fefb17d13882c8099899efd570f7e45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:12,266 INFO [StoreOpener-6fefb17d13882c8099899efd570f7e45-1 {}] regionserver.HStore(327): Store=6fefb17d13882c8099899efd570f7e45/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:12,266 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6fefb17d13882c8099899efd570f7e45 2024-11-27T12:28:12,267 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45 2024-11-27T12:28:12,270 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45 2024-11-27T12:28:12,271 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/recovered.edits/0000000000000000002 2024-11-27T12:28:12,279 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/recovered.edits/0000000000000000002 2024-11-27T12:28:12,282 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6fefb17d13882c8099899efd570f7e45 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-27T12:28:12,327 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/.tmp/a/ca2be54b8aae490eb487776a341c6440 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732710492066/Put/seqid=0 2024-11-27T12:28:12,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741844_1020 (size=5170) 2024-11-27T12:28:12,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741844_1020 (size=5170) 2024-11-27T12:28:12,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741844_1020 (size=5170) 2024-11-27T12:28:12,340 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/.tmp/a/ca2be54b8aae490eb487776a341c6440 2024-11-27T12:28:12,378 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/.tmp/a/ca2be54b8aae490eb487776a341c6440 as hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/a/ca2be54b8aae490eb487776a341c6440 2024-11-27T12:28:12,388 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/a/ca2be54b8aae490eb487776a341c6440, entries=2, sequenceid=2, filesize=5.0 K 2024-11-27T12:28:12,396 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 6fefb17d13882c8099899efd570f7e45 in 110ms, sequenceid=2, compaction requested=false; wal=null 2024-11-27T12:28:12,397 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/recovered.edits/0000000000000000002 2024-11-27T12:28:12,398 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6fefb17d13882c8099899efd570f7e45 2024-11-27T12:28:12,398 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6fefb17d13882c8099899efd570f7e45 2024-11-27T12:28:12,401 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6fefb17d13882c8099899efd570f7e45 2024-11-27T12:28:12,404 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/6fefb17d13882c8099899efd570f7e45/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-27T12:28:12,406 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6fefb17d13882c8099899efd570f7e45; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68941062, jitterRate=0.027301877737045288}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T12:28:12,407 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6fefb17d13882c8099899efd570f7e45: Writing region info on filesystem at 1732710492261Initializing all the Stores at 1732710492262 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710492262Obtaining lock to block concurrent updates at 1732710492282 (+20 ms)Preparing flush snapshotting stores in 6fefb17d13882c8099899efd570f7e45 at 1732710492282Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732710491987.6fefb17d13882c8099899efd570f7e45., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732710492285 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732710491987.6fefb17d13882c8099899efd570f7e45. at 1732710492285Flushing 6fefb17d13882c8099899efd570f7e45/a: creating writer at 1732710492286 (+1 ms)Flushing 6fefb17d13882c8099899efd570f7e45/a: appending metadata at 1732710492318 (+32 ms)Flushing 6fefb17d13882c8099899efd570f7e45/a: closing flushed file at 1732710492320 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@276c26bf: reopening flushed file at 1732710492377 (+57 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 6fefb17d13882c8099899efd570f7e45 in 110ms, sequenceid=2, compaction requested=false; wal=null at 1732710492396 (+19 ms)Cleaning up temporary data from old regions at 1732710492398 (+2 ms)Region opened successfully at 1732710492407 (+9 ms) 2024-11-27T12:28:12,436 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=379 (was 368) Potentially hanging thread: PacketResponder: BP-39017082-172.17.0.3-1732710484128:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:52354 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-39017082-172.17.0.3-1732710484128:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:39468 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-39017082-172.17.0.3-1732710484128:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:51412 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:52912 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:39664 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:38485/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=343 (was 343), ProcessCount=11 (was 11), AvailableMemoryMB=9182 (was 9252) 2024-11-27T12:28:12,446 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=379, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=343, ProcessCount=11, AvailableMemoryMB=9181 2024-11-27T12:28:12,462 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:12,464 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:12,465 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:28:12,468 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-91915671, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/hregion-91915671, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:12,482 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-91915671/hregion-91915671.1732710492469, exclude list is [], retry=0 2024-11-27T12:28:12,486 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:12,486 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:12,486 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:12,489 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-91915671/hregion-91915671.1732710492469 2024-11-27T12:28:12,490 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:46317:46317)] 2024-11-27T12:28:12,490 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 81b34ac9d0c1e61dd28f7316b4a90c47, NAME => 'testReplayEditsWrittenIntoWAL,,1732710492463.81b34ac9d0c1e61dd28f7316b4a90c47.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38485/hbase 2024-11-27T12:28:12,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741846_1022 (size=64) 2024-11-27T12:28:12,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741846_1022 (size=64) 2024-11-27T12:28:12,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741846_1022 (size=64) 2024-11-27T12:28:12,503 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732710492463.81b34ac9d0c1e61dd28f7316b4a90c47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:12,505 INFO [StoreOpener-81b34ac9d0c1e61dd28f7316b4a90c47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 81b34ac9d0c1e61dd28f7316b4a90c47 2024-11-27T12:28:12,507 INFO [StoreOpener-81b34ac9d0c1e61dd28f7316b4a90c47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81b34ac9d0c1e61dd28f7316b4a90c47 columnFamilyName a 2024-11-27T12:28:12,508 DEBUG [StoreOpener-81b34ac9d0c1e61dd28f7316b4a90c47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:12,508 INFO [StoreOpener-81b34ac9d0c1e61dd28f7316b4a90c47-1 {}] regionserver.HStore(327): Store=81b34ac9d0c1e61dd28f7316b4a90c47/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:12,509 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 81b34ac9d0c1e61dd28f7316b4a90c47 2024-11-27T12:28:12,510 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47 2024-11-27T12:28:12,510 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47 2024-11-27T12:28:12,511 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 81b34ac9d0c1e61dd28f7316b4a90c47 2024-11-27T12:28:12,511 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 81b34ac9d0c1e61dd28f7316b4a90c47 2024-11-27T12:28:12,513 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 81b34ac9d0c1e61dd28f7316b4a90c47 2024-11-27T12:28:12,517 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:28:12,517 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 81b34ac9d0c1e61dd28f7316b4a90c47; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61981315, jitterRate=-0.07640643417835236}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T12:28:12,518 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 81b34ac9d0c1e61dd28f7316b4a90c47: Writing region info on filesystem at 1732710492503Initializing all the Stores at 1732710492505 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710492505Cleaning up temporary data from old regions at 1732710492511 (+6 ms)Region opened successfully at 1732710492518 (+7 ms) 2024-11-27T12:28:12,518 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 81b34ac9d0c1e61dd28f7316b4a90c47, disabling compactions & flushes 2024-11-27T12:28:12,518 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732710492463.81b34ac9d0c1e61dd28f7316b4a90c47. 2024-11-27T12:28:12,518 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732710492463.81b34ac9d0c1e61dd28f7316b4a90c47. 2024-11-27T12:28:12,518 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732710492463.81b34ac9d0c1e61dd28f7316b4a90c47. after waiting 0 ms 2024-11-27T12:28:12,518 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732710492463.81b34ac9d0c1e61dd28f7316b4a90c47. 2024-11-27T12:28:12,518 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732710492463.81b34ac9d0c1e61dd28f7316b4a90c47. 2024-11-27T12:28:12,519 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 81b34ac9d0c1e61dd28f7316b4a90c47: Waiting for close lock at 1732710492518Disabling compacts and flushes for region at 1732710492518Disabling writes for close at 1732710492518Writing region close event to WAL at 1732710492518Closed at 1732710492518 2024-11-27T12:28:12,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741845_1021 (size=93) 2024-11-27T12:28:12,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741845_1021 (size=93) 2024-11-27T12:28:12,523 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-91915671/hregion-91915671.1732710492469 not finished, retry = 0 2024-11-27T12:28:12,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741845_1021 (size=93) 2024-11-27T12:28:12,628 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T12:28:12,628 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-91915671:(num 1732710492469) 2024-11-27T12:28:12,630 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-27T12:28:12,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741847_1023 (size=369) 2024-11-27T12:28:12,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741847_1023 (size=369) 2024-11-27T12:28:12,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741847_1023 (size=369) 2024-11-27T12:28:12,646 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-27T12:28:12,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741848_1024 (size=231) 2024-11-27T12:28:12,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741848_1024 (size=231) 2024-11-27T12:28:12,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741848_1024 (size=231) 2024-11-27T12:28:12,677 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-2, size=231 (231bytes) 2024-11-27T12:28:12,677 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-2 2024-11-27T12:28:12,678 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-2 after 1ms 2024-11-27T12:28:12,681 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-2 took 4ms 2024-11-27T12:28:12,683 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-2 so closing down 2024-11-27T12:28:12,683 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T12:28:12,686 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-27T12:28:12,687 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000002-wal-2.temp 2024-11-27T12:28:12,687 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T12:28:12,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741849_1025 (size=231) 2024-11-27T12:28:12,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741849_1025 (size=231) 2024-11-27T12:28:12,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741849_1025 (size=231) 2024-11-27T12:28:12,695 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-27T12:28:12,697 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000002 2024-11-27T12:28:12,697 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-27T12:28:12,697 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-2, journal: Splitting hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-2, size=231 (231bytes) at 1732710492677Finishing writing output for hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-2 so closing down at 1732710492683 (+6 ms)Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000002-wal-2.temp at 1732710492687 (+4 ms)3 split writer threads finished at 1732710492687Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732710492695 (+8 ms)Rename recovered edits hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000002 at 1732710492697 (+2 ms)Processed 1 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1732710492697 2024-11-27T12:28:12,713 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-1, size=369 (369bytes) 2024-11-27T12:28:12,713 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-1 2024-11-27T12:28:12,713 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-1 after 0ms 2024-11-27T12:28:12,717 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-1 took 5ms 2024-11-27T12:28:12,719 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-1 so closing down 2024-11-27T12:28:12,719 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T12:28:12,722 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-27T12:28:12,724 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000001-wal-1.temp 2024-11-27T12:28:12,724 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T12:28:12,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741850_1026 (size=369) 2024-11-27T12:28:12,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741850_1026 (size=369) 2024-11-27T12:28:12,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741850_1026 (size=369) 2024-11-27T12:28:12,731 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-27T12:28:12,736 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000002, length=231 2024-11-27T12:28:12,739 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000002 2024-11-27T12:28:12,739 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 22 ms; skipped=0; WAL=hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-27T12:28:12,739 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-1, journal: Splitting hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-1, size=369 (369bytes) at 1732710492713Finishing writing output for hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-1 so closing down at 1732710492719 (+6 ms)Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000001-wal-1.temp at 1732710492724 (+5 ms)3 split writer threads finished at 1732710492724Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732710492731 (+7 ms)Rename recovered edits hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000002 at 1732710492739 (+8 ms)Processed 2 edits across 1 Regions in 22 ms; skipped=0; WAL=hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1732710492739 2024-11-27T12:28:12,739 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:12,742 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:12,758 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal.1732710492742, exclude list is [], retry=0 2024-11-27T12:28:12,761 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:12,762 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:12,762 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:12,765 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal.1732710492742 2024-11-27T12:28:12,765 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:34427:34427)] 2024-11-27T12:28:12,765 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 81b34ac9d0c1e61dd28f7316b4a90c47, NAME => 'testReplayEditsWrittenIntoWAL,,1732710492463.81b34ac9d0c1e61dd28f7316b4a90c47.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:12,766 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732710492463.81b34ac9d0c1e61dd28f7316b4a90c47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:12,766 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 81b34ac9d0c1e61dd28f7316b4a90c47 2024-11-27T12:28:12,766 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 81b34ac9d0c1e61dd28f7316b4a90c47 2024-11-27T12:28:12,768 INFO [StoreOpener-81b34ac9d0c1e61dd28f7316b4a90c47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 81b34ac9d0c1e61dd28f7316b4a90c47 2024-11-27T12:28:12,770 INFO [StoreOpener-81b34ac9d0c1e61dd28f7316b4a90c47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81b34ac9d0c1e61dd28f7316b4a90c47 columnFamilyName a 2024-11-27T12:28:12,771 DEBUG [StoreOpener-81b34ac9d0c1e61dd28f7316b4a90c47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:12,771 INFO [StoreOpener-81b34ac9d0c1e61dd28f7316b4a90c47-1 {}] regionserver.HStore(327): Store=81b34ac9d0c1e61dd28f7316b4a90c47/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:12,772 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 81b34ac9d0c1e61dd28f7316b4a90c47 2024-11-27T12:28:12,773 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47 2024-11-27T12:28:12,775 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47 2024-11-27T12:28:12,777 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000002 2024-11-27T12:28:12,782 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000002 2024-11-27T12:28:12,782 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 81b34ac9d0c1e61dd28f7316b4a90c47 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-27T12:28:12,797 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/.tmp/a/59483ab8f74f4941af881089494c6677 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732710492628/Put/seqid=0 2024-11-27T12:28:12,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741852_1028 (size=5170) 2024-11-27T12:28:12,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741852_1028 (size=5170) 2024-11-27T12:28:12,807 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/.tmp/a/59483ab8f74f4941af881089494c6677 2024-11-27T12:28:12,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741852_1028 (size=5170) 2024-11-27T12:28:12,817 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/.tmp/a/59483ab8f74f4941af881089494c6677 as hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/a/59483ab8f74f4941af881089494c6677 2024-11-27T12:28:12,825 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/a/59483ab8f74f4941af881089494c6677, entries=2, sequenceid=2, filesize=5.0 K 2024-11-27T12:28:12,826 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 81b34ac9d0c1e61dd28f7316b4a90c47 in 43ms, sequenceid=2, compaction requested=false; wal=null 2024-11-27T12:28:12,826 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/0000000000000000002 2024-11-27T12:28:12,827 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 81b34ac9d0c1e61dd28f7316b4a90c47 2024-11-27T12:28:12,827 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 81b34ac9d0c1e61dd28f7316b4a90c47 2024-11-27T12:28:12,829 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 81b34ac9d0c1e61dd28f7316b4a90c47 2024-11-27T12:28:12,832 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/81b34ac9d0c1e61dd28f7316b4a90c47/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-27T12:28:12,833 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 81b34ac9d0c1e61dd28f7316b4a90c47; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66548679, jitterRate=-0.00834740698337555}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T12:28:12,833 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 81b34ac9d0c1e61dd28f7316b4a90c47: Writing region info on filesystem at 1732710492766Initializing all the Stores at 1732710492768 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710492768Obtaining lock to block concurrent updates at 1732710492782 (+14 ms)Preparing flush snapshotting stores in 81b34ac9d0c1e61dd28f7316b4a90c47 at 1732710492782Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732710492463.81b34ac9d0c1e61dd28f7316b4a90c47., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732710492782Flushing stores of testReplayEditsWrittenIntoWAL,,1732710492463.81b34ac9d0c1e61dd28f7316b4a90c47. at 1732710492782Flushing 81b34ac9d0c1e61dd28f7316b4a90c47/a: creating writer at 1732710492782Flushing 81b34ac9d0c1e61dd28f7316b4a90c47/a: appending metadata at 1732710492796 (+14 ms)Flushing 81b34ac9d0c1e61dd28f7316b4a90c47/a: closing flushed file at 1732710492796Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7019a6d1: reopening flushed file at 1732710492816 (+20 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 81b34ac9d0c1e61dd28f7316b4a90c47 in 43ms, sequenceid=2, compaction requested=false; wal=null at 1732710492826 (+10 ms)Cleaning up temporary data from old regions at 1732710492827 (+1 ms)Region opened successfully at 1732710492833 (+6 ms) 2024-11-27T12:28:12,848 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=389 (was 379) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:52426 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:47212 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-39017082-172.17.0.3-1732710484128:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:39664 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-39017082-172.17.0.3-1732710484128:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:52984 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:51412 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:39528 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-39017082-172.17.0.3-1732710484128:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=343 (was 343), ProcessCount=11 (was 11), AvailableMemoryMB=9176 (was 9181) 2024-11-27T12:28:12,861 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=389, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=343, ProcessCount=11, AvailableMemoryMB=9175 2024-11-27T12:28:12,878 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:12,880 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:12,881 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:28:12,884 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-35541299, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/hregion-35541299, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:12,895 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-35541299/hregion-35541299.1732710492884, exclude list is [], retry=0 2024-11-27T12:28:12,899 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:12,899 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:12,899 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:12,901 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-35541299/hregion-35541299.1732710492884 2024-11-27T12:28:12,902 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:46317:46317)] 2024-11-27T12:28:12,902 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 329c6df3f79f38efaf66991c1224cc63, NAME => 'testReplayEditsWrittenIntoWAL,,1732710492879.329c6df3f79f38efaf66991c1224cc63.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38485/hbase 2024-11-27T12:28:12,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741854_1030 (size=64) 2024-11-27T12:28:12,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741854_1030 (size=64) 2024-11-27T12:28:12,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741854_1030 (size=64) 2024-11-27T12:28:12,913 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732710492879.329c6df3f79f38efaf66991c1224cc63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:12,915 INFO [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 329c6df3f79f38efaf66991c1224cc63 2024-11-27T12:28:12,916 INFO [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 329c6df3f79f38efaf66991c1224cc63 columnFamilyName a 2024-11-27T12:28:12,916 DEBUG [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:12,917 INFO [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] regionserver.HStore(327): Store=329c6df3f79f38efaf66991c1224cc63/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:12,917 INFO [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 329c6df3f79f38efaf66991c1224cc63 2024-11-27T12:28:12,919 INFO [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 329c6df3f79f38efaf66991c1224cc63 columnFamilyName b 2024-11-27T12:28:12,919 DEBUG [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:12,919 INFO [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] regionserver.HStore(327): Store=329c6df3f79f38efaf66991c1224cc63/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:12,919 INFO [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 329c6df3f79f38efaf66991c1224cc63 2024-11-27T12:28:12,921 INFO [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 329c6df3f79f38efaf66991c1224cc63 columnFamilyName c 2024-11-27T12:28:12,921 DEBUG [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:12,921 INFO [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] regionserver.HStore(327): Store=329c6df3f79f38efaf66991c1224cc63/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:12,922 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 329c6df3f79f38efaf66991c1224cc63 2024-11-27T12:28:12,923 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63 2024-11-27T12:28:12,923 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63 2024-11-27T12:28:12,924 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 329c6df3f79f38efaf66991c1224cc63 2024-11-27T12:28:12,924 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 329c6df3f79f38efaf66991c1224cc63 2024-11-27T12:28:12,925 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T12:28:12,926 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 329c6df3f79f38efaf66991c1224cc63 2024-11-27T12:28:12,929 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:28:12,930 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 329c6df3f79f38efaf66991c1224cc63; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63889138, jitterRate=-0.04797765612602234}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T12:28:12,930 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 329c6df3f79f38efaf66991c1224cc63: Writing region info on filesystem at 1732710492913Initializing all the Stores at 1732710492914 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710492914Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710492915 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710492915Cleaning up temporary data from old regions at 1732710492924 (+9 ms)Region opened successfully at 1732710492930 (+6 ms) 2024-11-27T12:28:12,931 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 329c6df3f79f38efaf66991c1224cc63, disabling compactions & flushes 2024-11-27T12:28:12,931 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732710492879.329c6df3f79f38efaf66991c1224cc63. 2024-11-27T12:28:12,931 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732710492879.329c6df3f79f38efaf66991c1224cc63. 2024-11-27T12:28:12,931 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732710492879.329c6df3f79f38efaf66991c1224cc63. after waiting 0 ms 2024-11-27T12:28:12,931 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732710492879.329c6df3f79f38efaf66991c1224cc63. 2024-11-27T12:28:12,931 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732710492879.329c6df3f79f38efaf66991c1224cc63. 2024-11-27T12:28:12,932 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 329c6df3f79f38efaf66991c1224cc63: Waiting for close lock at 1732710492931Disabling compacts and flushes for region at 1732710492931Disabling writes for close at 1732710492931Writing region close event to WAL at 1732710492931Closed at 1732710492931 2024-11-27T12:28:12,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741853_1029 (size=93) 2024-11-27T12:28:12,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741853_1029 (size=93) 2024-11-27T12:28:12,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741853_1029 (size=93) 2024-11-27T12:28:12,938 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T12:28:12,939 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-35541299:(num 1732710492884) 2024-11-27T12:28:12,939 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:12,942 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:12,957 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710492942, exclude list is [], retry=0 2024-11-27T12:28:12,961 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:12,961 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:12,962 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:12,964 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710492942 2024-11-27T12:28:12,964 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:46317:46317)] 2024-11-27T12:28:13,157 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710492942, size=0 (0bytes) 2024-11-27T12:28:13,157 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710492942 might be still open, length is 0 2024-11-27T12:28:13,157 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710492942 2024-11-27T12:28:13,158 WARN [IPC Server handler 3 on default port 38485 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710492942 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-27T12:28:13,158 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710492942 after 1ms 2024-11-27T12:28:13,767 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:53024 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53024 dst: /127.0.0.1:37469 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37469 remote=/127.0.0.1:53024]. Total timeout mills is 60000, 59358 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:13,768 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:39562 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:38877:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39562 dst: /127.0.0.1:38877 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:13,768 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:52476 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:38365:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52476 dst: /127.0.0.1:38365 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:13,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741855_1032 (size=470675) 2024-11-27T12:28:13,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741855_1032 (size=470675) 2024-11-27T12:28:13,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741855_1032 (size=470675) 2024-11-27T12:28:17,106 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T12:28:17,159 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710492942 after 4002ms 2024-11-27T12:28:17,159 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T12:28:17,165 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710492942 took 4009ms 2024-11-27T12:28:17,169 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732710492942.temp 2024-11-27T12:28:17,171 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/recovered.edits/0000000000000000001-wal.1732710492942.temp 2024-11-27T12:28:17,247 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710492942; continuing. 2024-11-27T12:28:17,247 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710492942 so closing down 2024-11-27T12:28:17,247 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T12:28:17,248 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T12:28:17,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741856_1033 (size=470683) 2024-11-27T12:28:17,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741856_1033 (size=470683) 2024-11-27T12:28:17,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741856_1033 (size=470683) 2024-11-27T12:28:17,251 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/recovered.edits/0000000000000000001-wal.1732710492942.temp (wrote 3002 edits, skipped 0 edits in 32 ms) 2024-11-27T12:28:17,253 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/recovered.edits/0000000000000000001-wal.1732710492942.temp to hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/recovered.edits/0000000000000003002 2024-11-27T12:28:17,253 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 88 ms; skipped=0; WAL=hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710492942, size=0, length=0, corrupted=false, cancelled=false 2024-11-27T12:28:17,253 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710492942, journal: Splitting hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710492942, size=0 (0bytes) at 1732710493157Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/recovered.edits/0000000000000000001-wal.1732710492942.temp at 1732710497171 (+4014 ms)Split 1024 edits, skipped 0 edits. at 1732710497213 (+42 ms)Split 2048 edits, skipped 0 edits. at 1732710497232 (+19 ms)Finishing writing output for hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710492942 so closing down at 1732710497247 (+15 ms)3 split writer threads finished at 1732710497248 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/recovered.edits/0000000000000000001-wal.1732710492942.temp (wrote 3002 edits, skipped 0 edits in 32 ms) at 1732710497251 (+3 ms)Rename recovered edits hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/recovered.edits/0000000000000000001-wal.1732710492942.temp to hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/recovered.edits/0000000000000003002 at 1732710497253 (+2 ms)Processed 3002 edits across 1 Regions in 88 ms; skipped=0; WAL=hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710492942, size=0, length=0, corrupted=false, cancelled=false at 1732710497253 2024-11-27T12:28:17,256 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710492942 to hdfs://localhost:38485/hbase/oldWALs/wal.1732710492942 2024-11-27T12:28:17,257 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/recovered.edits/0000000000000003002 2024-11-27T12:28:17,257 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:17,260 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:17,279 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710497260, exclude list is [], retry=0 2024-11-27T12:28:17,282 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:17,283 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:17,283 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:17,286 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710497260 2024-11-27T12:28:17,286 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:45591:45591)] 2024-11-27T12:28:17,287 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732710492879.329c6df3f79f38efaf66991c1224cc63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:17,289 INFO [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 329c6df3f79f38efaf66991c1224cc63 2024-11-27T12:28:17,290 INFO [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 329c6df3f79f38efaf66991c1224cc63 columnFamilyName a 2024-11-27T12:28:17,291 DEBUG [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:17,291 INFO [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] regionserver.HStore(327): Store=329c6df3f79f38efaf66991c1224cc63/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:17,291 INFO [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 329c6df3f79f38efaf66991c1224cc63 2024-11-27T12:28:17,292 INFO [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 329c6df3f79f38efaf66991c1224cc63 columnFamilyName b 2024-11-27T12:28:17,292 DEBUG [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:17,293 INFO [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] regionserver.HStore(327): Store=329c6df3f79f38efaf66991c1224cc63/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:17,293 INFO [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 329c6df3f79f38efaf66991c1224cc63 2024-11-27T12:28:17,294 INFO [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 329c6df3f79f38efaf66991c1224cc63 columnFamilyName c 2024-11-27T12:28:17,294 DEBUG [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:17,295 INFO [StoreOpener-329c6df3f79f38efaf66991c1224cc63-1 {}] regionserver.HStore(327): Store=329c6df3f79f38efaf66991c1224cc63/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:17,295 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 329c6df3f79f38efaf66991c1224cc63 2024-11-27T12:28:17,296 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63 2024-11-27T12:28:17,298 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63 2024-11-27T12:28:17,299 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/recovered.edits/0000000000000003002 2024-11-27T12:28:17,344 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-27T12:28:17,684 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 329c6df3f79f38efaf66991c1224cc63 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-27T12:28:17,724 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/.tmp/a/512e11418596417384800935697ca272 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732710492973/Put/seqid=0 2024-11-27T12:28:17,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741858_1035 (size=50463) 2024-11-27T12:28:17,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741858_1035 (size=50463) 2024-11-27T12:28:17,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741858_1035 (size=50463) 2024-11-27T12:28:17,732 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/.tmp/a/512e11418596417384800935697ca272 2024-11-27T12:28:17,739 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/.tmp/a/512e11418596417384800935697ca272 as hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/a/512e11418596417384800935697ca272 2024-11-27T12:28:17,747 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/a/512e11418596417384800935697ca272, entries=754, sequenceid=754, filesize=49.3 K 2024-11-27T12:28:17,747 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 329c6df3f79f38efaf66991c1224cc63 in 63ms, sequenceid=754, compaction requested=false; wal=null 2024-11-27T12:28:17,760 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-27T12:28:17,760 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 329c6df3f79f38efaf66991c1224cc63 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-27T12:28:17,770 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/.tmp/a/c552ed0c7ab747e1aef4e3daf3103d0d is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732710493022/Put/seqid=0 2024-11-27T12:28:17,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741859_1036 (size=20072) 2024-11-27T12:28:17,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741859_1036 (size=20072) 2024-11-27T12:28:17,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741859_1036 (size=20072) 2024-11-27T12:28:17,779 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/.tmp/a/c552ed0c7ab747e1aef4e3daf3103d0d 2024-11-27T12:28:17,805 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/.tmp/b/99ea4b44e7894e7eb6fc0e7b8daeaf98 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732710493038/Put/seqid=0 2024-11-27T12:28:17,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741860_1037 (size=35835) 2024-11-27T12:28:17,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741860_1037 (size=35835) 2024-11-27T12:28:17,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741860_1037 (size=35835) 2024-11-27T12:28:17,813 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/.tmp/b/99ea4b44e7894e7eb6fc0e7b8daeaf98 2024-11-27T12:28:17,821 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/.tmp/a/c552ed0c7ab747e1aef4e3daf3103d0d as hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/a/c552ed0c7ab747e1aef4e3daf3103d0d 2024-11-27T12:28:17,829 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/a/c552ed0c7ab747e1aef4e3daf3103d0d, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-27T12:28:17,830 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/.tmp/b/99ea4b44e7894e7eb6fc0e7b8daeaf98 as hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/b/99ea4b44e7894e7eb6fc0e7b8daeaf98 2024-11-27T12:28:17,837 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/b/99ea4b44e7894e7eb6fc0e7b8daeaf98, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-27T12:28:17,838 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 329c6df3f79f38efaf66991c1224cc63 in 78ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-27T12:28:17,848 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-27T12:28:17,848 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 329c6df3f79f38efaf66991c1224cc63 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-27T12:28:17,855 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/.tmp/b/ed688f6e1e174e7ba176440cd0498ffb is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732710493055/Put/seqid=0 2024-11-27T12:28:17,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741861_1038 (size=35082) 2024-11-27T12:28:17,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741861_1038 (size=35082) 2024-11-27T12:28:17,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741861_1038 (size=35082) 2024-11-27T12:28:17,868 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/.tmp/b/ed688f6e1e174e7ba176440cd0498ffb 2024-11-27T12:28:17,890 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/.tmp/c/9bfb44eb2b144603a9975db03a095bfb is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732710493083/Put/seqid=0 2024-11-27T12:28:17,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741862_1039 (size=20825) 2024-11-27T12:28:17,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741862_1039 (size=20825) 2024-11-27T12:28:17,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741862_1039 (size=20825) 2024-11-27T12:28:17,899 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/.tmp/c/9bfb44eb2b144603a9975db03a095bfb 2024-11-27T12:28:17,908 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/.tmp/b/ed688f6e1e174e7ba176440cd0498ffb as hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/b/ed688f6e1e174e7ba176440cd0498ffb 2024-11-27T12:28:17,914 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/b/ed688f6e1e174e7ba176440cd0498ffb, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-27T12:28:17,916 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/.tmp/c/9bfb44eb2b144603a9975db03a095bfb as hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/c/9bfb44eb2b144603a9975db03a095bfb 2024-11-27T12:28:17,924 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/c/9bfb44eb2b144603a9975db03a095bfb, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-27T12:28:17,924 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 329c6df3f79f38efaf66991c1224cc63 in 76ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-27T12:28:17,936 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732710493123/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732710492879.329c6df3f79f38efaf66991c1224cc63. 2024-11-27T12:28:17,940 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/recovered.edits/0000000000000003002 2024-11-27T12:28:17,941 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-27T12:28:17,941 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 329c6df3f79f38efaf66991c1224cc63 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-27T12:28:17,952 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/.tmp/c/a906393f4ae24900b968708cb65eb03f is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732710493090/Put/seqid=0 2024-11-27T12:28:17,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741863_1040 (size=50301) 2024-11-27T12:28:17,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741863_1040 (size=50301) 2024-11-27T12:28:17,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741863_1040 (size=50301) 2024-11-27T12:28:17,962 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/.tmp/c/a906393f4ae24900b968708cb65eb03f 2024-11-27T12:28:17,969 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a906393f4ae24900b968708cb65eb03f 2024-11-27T12:28:17,971 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/.tmp/c/a906393f4ae24900b968708cb65eb03f as hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/c/a906393f4ae24900b968708cb65eb03f 2024-11-27T12:28:17,977 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a906393f4ae24900b968708cb65eb03f 2024-11-27T12:28:17,977 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/c/a906393f4ae24900b968708cb65eb03f, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-27T12:28:17,978 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 329c6df3f79f38efaf66991c1224cc63 in 37ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-27T12:28:17,979 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/recovered.edits/0000000000000003002 2024-11-27T12:28:17,980 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 329c6df3f79f38efaf66991c1224cc63 2024-11-27T12:28:17,980 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 329c6df3f79f38efaf66991c1224cc63 2024-11-27T12:28:17,981 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T12:28:17,983 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 329c6df3f79f38efaf66991c1224cc63 2024-11-27T12:28:17,985 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenIntoWAL/329c6df3f79f38efaf66991c1224cc63/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-27T12:28:17,986 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 329c6df3f79f38efaf66991c1224cc63; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66749343, jitterRate=-0.005357280373573303}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T12:28:17,987 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 329c6df3f79f38efaf66991c1224cc63: Writing region info on filesystem at 1732710497287Initializing all the Stores at 1732710497288 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710497289 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710497289Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710497289Cleaning up temporary data from old regions at 1732710497980 (+691 ms)Region opened successfully at 1732710497986 (+6 ms) 2024-11-27T12:28:18,060 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 329c6df3f79f38efaf66991c1224cc63, disabling compactions & flushes 2024-11-27T12:28:18,060 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732710492879.329c6df3f79f38efaf66991c1224cc63. 2024-11-27T12:28:18,060 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732710492879.329c6df3f79f38efaf66991c1224cc63. 2024-11-27T12:28:18,060 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732710492879.329c6df3f79f38efaf66991c1224cc63. after waiting 0 ms 2024-11-27T12:28:18,060 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732710492879.329c6df3f79f38efaf66991c1224cc63. 2024-11-27T12:28:18,062 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732710492879.329c6df3f79f38efaf66991c1224cc63. 2024-11-27T12:28:18,062 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 329c6df3f79f38efaf66991c1224cc63: Waiting for close lock at 1732710498060Disabling compacts and flushes for region at 1732710498060Disabling writes for close at 1732710498060Writing region close event to WAL at 1732710498062 (+2 ms)Closed at 1732710498062 2024-11-27T12:28:18,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741857_1034 (size=93) 2024-11-27T12:28:18,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741857_1034 (size=93) 2024-11-27T12:28:18,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741857_1034 (size=93) 2024-11-27T12:28:18,068 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T12:28:18,068 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732710497260) 2024-11-27T12:28:18,082 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=403 (was 389) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-738645004_22 at /127.0.0.1:52554 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-738645004_22 at /127.0.0.1:52536 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36597 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1612229954) connection to localhost/127.0.0.1:36597 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-738645004_22 at /127.0.0.1:53064 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40089 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-738645004_22 at /127.0.0.1:39614 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1612229954) connection to localhost/127.0.0.1:38485 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1612229954) connection to localhost/127.0.0.1:40089 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:38485 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=324 (was 343), ProcessCount=11 (was 11), AvailableMemoryMB=9120 (was 9175) 2024-11-27T12:28:18,095 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=403, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=324, ProcessCount=11, AvailableMemoryMB=9120 2024-11-27T12:28:18,118 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:18,121 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:18,122 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:28:18,126 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-76418173, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/hregion-76418173, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:18,140 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-76418173/hregion-76418173.1732710498127, exclude list is [], retry=0 2024-11-27T12:28:18,143 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:18,143 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:18,143 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:18,146 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-76418173/hregion-76418173.1732710498127 2024-11-27T12:28:18,147 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:45591:45591)] 2024-11-27T12:28:18,147 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 47fcc6b572c37f2e5eeb2f9727848931, NAME => 'test2727,,1732710498119.47fcc6b572c37f2e5eeb2f9727848931.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38485/hbase 2024-11-27T12:28:18,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741865_1042 (size=43) 2024-11-27T12:28:18,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741865_1042 (size=43) 2024-11-27T12:28:18,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741865_1042 (size=43) 2024-11-27T12:28:18,159 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732710498119.47fcc6b572c37f2e5eeb2f9727848931.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:18,160 INFO [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,162 INFO [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47fcc6b572c37f2e5eeb2f9727848931 columnFamilyName a 2024-11-27T12:28:18,162 DEBUG [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:18,162 INFO [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] regionserver.HStore(327): Store=47fcc6b572c37f2e5eeb2f9727848931/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:18,162 INFO [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,164 INFO [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47fcc6b572c37f2e5eeb2f9727848931 columnFamilyName b 2024-11-27T12:28:18,164 DEBUG [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:18,165 INFO [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] regionserver.HStore(327): Store=47fcc6b572c37f2e5eeb2f9727848931/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:18,165 INFO [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,166 INFO [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47fcc6b572c37f2e5eeb2f9727848931 columnFamilyName c 2024-11-27T12:28:18,166 DEBUG [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:18,167 INFO [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] regionserver.HStore(327): Store=47fcc6b572c37f2e5eeb2f9727848931/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:18,167 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,168 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,168 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,169 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,169 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,170 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T12:28:18,171 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,174 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:28:18,175 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 47fcc6b572c37f2e5eeb2f9727848931; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69927161, jitterRate=0.04199589788913727}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T12:28:18,176 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 47fcc6b572c37f2e5eeb2f9727848931: Writing region info on filesystem at 1732710498159Initializing all the Stores at 1732710498160 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710498160Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710498160Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710498160Cleaning up temporary data from old regions at 1732710498169 (+9 ms)Region opened successfully at 1732710498176 (+7 ms) 2024-11-27T12:28:18,177 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 47fcc6b572c37f2e5eeb2f9727848931, disabling compactions & flushes 2024-11-27T12:28:18,177 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732710498119.47fcc6b572c37f2e5eeb2f9727848931. 2024-11-27T12:28:18,177 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732710498119.47fcc6b572c37f2e5eeb2f9727848931. 2024-11-27T12:28:18,177 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732710498119.47fcc6b572c37f2e5eeb2f9727848931. after waiting 0 ms 2024-11-27T12:28:18,177 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732710498119.47fcc6b572c37f2e5eeb2f9727848931. 2024-11-27T12:28:18,177 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732710498119.47fcc6b572c37f2e5eeb2f9727848931. 2024-11-27T12:28:18,178 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 47fcc6b572c37f2e5eeb2f9727848931: Waiting for close lock at 1732710498177Disabling compacts and flushes for region at 1732710498177Disabling writes for close at 1732710498177Writing region close event to WAL at 1732710498177Closed at 1732710498177 2024-11-27T12:28:18,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741864_1041 (size=93) 2024-11-27T12:28:18,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741864_1041 (size=93) 2024-11-27T12:28:18,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741864_1041 (size=93) 2024-11-27T12:28:18,184 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T12:28:18,184 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-76418173:(num 1732710498127) 2024-11-27T12:28:18,184 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:18,186 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:18,198 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498187, exclude list is [], retry=0 2024-11-27T12:28:18,202 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:18,203 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:18,203 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:18,206 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498187 2024-11-27T12:28:18,207 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:45591:45591)] 2024-11-27T12:28:18,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741866_1043 (size=344306) 2024-11-27T12:28:18,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741866_1043 (size=344306) 2024-11-27T12:28:18,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741866_1043 (size=344306) 2024-11-27T12:28:18,385 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498187, size=336.2 K (344306bytes) 2024-11-27T12:28:18,385 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498187 2024-11-27T12:28:18,386 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498187 after 1ms 2024-11-27T12:28:18,390 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498187 took 5ms 2024-11-27T12:28:18,395 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732710498187.temp 2024-11-27T12:28:18,397 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000000001-wal.1732710498187.temp 2024-11-27T12:28:18,417 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498187 so closing down 2024-11-27T12:28:18,417 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T12:28:18,417 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T12:28:18,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741867_1044 (size=344306) 2024-11-27T12:28:18,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741867_1044 (size=344306) 2024-11-27T12:28:18,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741867_1044 (size=344306) 2024-11-27T12:28:18,421 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000000001-wal.1732710498187.temp (wrote 3000 edits, skipped 0 edits in 10 ms) 2024-11-27T12:28:18,423 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000000001-wal.1732710498187.temp to hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000003000 2024-11-27T12:28:18,423 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 32 ms; skipped=0; WAL=hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498187, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-11-27T12:28:18,424 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498187, journal: Splitting hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498187, size=336.2 K (344306bytes) at 1732710498385Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000000001-wal.1732710498187.temp at 1732710498397 (+12 ms)Split 1024 edits, skipped 0 edits. at 1732710498400 (+3 ms)Split 2048 edits, skipped 0 edits. at 1732710498409 (+9 ms)Finishing writing output for hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498187 so closing down at 1732710498417 (+8 ms)3 split writer threads finished at 1732710498417Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000000001-wal.1732710498187.temp (wrote 3000 edits, skipped 0 edits in 10 ms) at 1732710498421 (+4 ms)Rename recovered edits hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000000001-wal.1732710498187.temp to hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000003000 at 1732710498423 (+2 ms)Processed 3000 edits across 1 Regions in 32 ms; skipped=0; WAL=hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498187, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1732710498423 2024-11-27T12:28:18,426 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498187 to hdfs://localhost:38485/hbase/oldWALs/wal.1732710498187 2024-11-27T12:28:18,427 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000003000 2024-11-27T12:28:18,427 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:18,430 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:18,444 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498430, exclude list is [], retry=0 2024-11-27T12:28:18,448 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:18,448 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:18,449 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:18,451 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498430 2024-11-27T12:28:18,452 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:45591:45591)] 2024-11-27T12:28:18,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741868_1045 (size=344433) 2024-11-27T12:28:18,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741868_1045 (size=344433) 2024-11-27T12:28:18,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741868_1045 (size=344433) 2024-11-27T12:28:18,620 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498430, size=336.4 K (344433bytes) 2024-11-27T12:28:18,620 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498430 2024-11-27T12:28:18,621 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498430 after 1ms 2024-11-27T12:28:18,625 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498430 took 5ms 2024-11-27T12:28:18,630 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732710498430.temp 2024-11-27T12:28:18,631 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000003001-wal.1732710498430.temp 2024-11-27T12:28:18,660 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498430 so closing down 2024-11-27T12:28:18,660 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T12:28:18,660 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T12:28:18,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741869_1046 (size=344433) 2024-11-27T12:28:18,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741869_1046 (size=344433) 2024-11-27T12:28:18,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741869_1046 (size=344433) 2024-11-27T12:28:18,664 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000003001-wal.1732710498430.temp (wrote 3000 edits, skipped 0 edits in 11 ms) 2024-11-27T12:28:18,666 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000003001-wal.1732710498430.temp to hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000006000 2024-11-27T12:28:18,666 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498430, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-11-27T12:28:18,666 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498430, journal: Splitting hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498430, size=336.4 K (344433bytes) at 1732710498620Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000003001-wal.1732710498430.temp at 1732710498631 (+11 ms)Split 1024 edits, skipped 0 edits. at 1732710498637 (+6 ms)Split 2048 edits, skipped 0 edits. at 1732710498649 (+12 ms)Finishing writing output for hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498430 so closing down at 1732710498660 (+11 ms)3 split writer threads finished at 1732710498660Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000003001-wal.1732710498430.temp (wrote 3000 edits, skipped 0 edits in 11 ms) at 1732710498664 (+4 ms)Rename recovered edits hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000003001-wal.1732710498430.temp to hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000006000 at 1732710498666 (+2 ms)Processed 3000 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498430, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1732710498666 2024-11-27T12:28:18,669 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498430 to hdfs://localhost:38485/hbase/oldWALs/wal.1732710498430 2024-11-27T12:28:18,670 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000006000 2024-11-27T12:28:18,670 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:18,672 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/test2727-manual,16010,1732710498117, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:18,694 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498673, exclude list is [], retry=0 2024-11-27T12:28:18,697 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:18,698 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:18,698 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:18,700 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732710498117/wal.1732710498673 2024-11-27T12:28:18,702 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:34427:34427)] 2024-11-27T12:28:18,702 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 47fcc6b572c37f2e5eeb2f9727848931, NAME => 'test2727,,1732710498119.47fcc6b572c37f2e5eeb2f9727848931.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:18,702 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732710498119.47fcc6b572c37f2e5eeb2f9727848931.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:18,703 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,703 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,704 INFO [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,706 INFO [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47fcc6b572c37f2e5eeb2f9727848931 columnFamilyName a 2024-11-27T12:28:18,706 DEBUG [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:18,707 INFO [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] regionserver.HStore(327): Store=47fcc6b572c37f2e5eeb2f9727848931/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:18,707 INFO [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,708 INFO [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47fcc6b572c37f2e5eeb2f9727848931 columnFamilyName b 2024-11-27T12:28:18,708 DEBUG [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:18,709 INFO [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] regionserver.HStore(327): Store=47fcc6b572c37f2e5eeb2f9727848931/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:18,709 INFO [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,710 INFO [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47fcc6b572c37f2e5eeb2f9727848931 columnFamilyName c 2024-11-27T12:28:18,710 DEBUG [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:18,710 INFO [StoreOpener-47fcc6b572c37f2e5eeb2f9727848931-1 {}] regionserver.HStore(327): Store=47fcc6b572c37f2e5eeb2f9727848931/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:18,710 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,711 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,714 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,715 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000003000 2024-11-27T12:28:18,765 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000003000 2024-11-27T12:28:18,766 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000006000 2024-11-27T12:28:18,804 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000006000 2024-11-27T12:28:18,805 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 47fcc6b572c37f2e5eeb2f9727848931 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-27T12:28:18,826 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/.tmp/a/278d56962e53426c8605fcf017c5025d is 41, key is test2727/a:100/1732710498456/Put/seqid=0 2024-11-27T12:28:18,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741871_1048 (size=84227) 2024-11-27T12:28:18,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741871_1048 (size=84227) 2024-11-27T12:28:18,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741871_1048 (size=84227) 2024-11-27T12:28:18,833 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/.tmp/a/278d56962e53426c8605fcf017c5025d 2024-11-27T12:28:18,862 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/.tmp/b/73ca89a70c564ceba1c07cd8d4b5d3d1 is 41, key is test2727/b:100/1732710498509/Put/seqid=0 2024-11-27T12:28:18,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741872_1049 (size=84609) 2024-11-27T12:28:18,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741872_1049 (size=84609) 2024-11-27T12:28:18,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741872_1049 (size=84609) 2024-11-27T12:28:18,869 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/.tmp/b/73ca89a70c564ceba1c07cd8d4b5d3d1 2024-11-27T12:28:18,906 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/.tmp/c/0ed214a00e06446ab15b07331c9ae8e9 is 41, key is test2727/c:100/1732710498556/Put/seqid=0 2024-11-27T12:28:18,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741873_1050 (size=84609) 2024-11-27T12:28:18,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741873_1050 (size=84609) 2024-11-27T12:28:18,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741873_1050 (size=84609) 2024-11-27T12:28:18,913 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/.tmp/c/0ed214a00e06446ab15b07331c9ae8e9 2024-11-27T12:28:18,920 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/.tmp/a/278d56962e53426c8605fcf017c5025d as hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/a/278d56962e53426c8605fcf017c5025d 2024-11-27T12:28:18,926 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/a/278d56962e53426c8605fcf017c5025d, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-27T12:28:18,928 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/.tmp/b/73ca89a70c564ceba1c07cd8d4b5d3d1 as hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/b/73ca89a70c564ceba1c07cd8d4b5d3d1 2024-11-27T12:28:18,934 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/b/73ca89a70c564ceba1c07cd8d4b5d3d1, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-27T12:28:18,935 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/.tmp/c/0ed214a00e06446ab15b07331c9ae8e9 as hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/c/0ed214a00e06446ab15b07331c9ae8e9 2024-11-27T12:28:18,942 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/c/0ed214a00e06446ab15b07331c9ae8e9, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-27T12:28:18,942 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 47fcc6b572c37f2e5eeb2f9727848931 in 138ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-27T12:28:18,943 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000003000 2024-11-27T12:28:18,944 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/0000000000000006000 2024-11-27T12:28:18,945 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,945 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,946 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T12:28:18,948 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 47fcc6b572c37f2e5eeb2f9727848931 2024-11-27T12:28:18,950 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/test2727/47fcc6b572c37f2e5eeb2f9727848931/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-27T12:28:18,952 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 47fcc6b572c37f2e5eeb2f9727848931; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68773455, jitterRate=0.024804338812828064}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T12:28:18,953 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 47fcc6b572c37f2e5eeb2f9727848931: Writing region info on filesystem at 1732710498703Initializing all the Stores at 1732710498704 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710498704Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710498704Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710498704Obtaining lock to block concurrent updates at 1732710498805 (+101 ms)Preparing flush snapshotting stores in 47fcc6b572c37f2e5eeb2f9727848931 at 1732710498805Finished memstore snapshotting test2727,,1732710498119.47fcc6b572c37f2e5eeb2f9727848931., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732710498805Flushing stores of test2727,,1732710498119.47fcc6b572c37f2e5eeb2f9727848931. at 1732710498805Flushing 47fcc6b572c37f2e5eeb2f9727848931/a: creating writer at 1732710498805Flushing 47fcc6b572c37f2e5eeb2f9727848931/a: appending metadata at 1732710498825 (+20 ms)Flushing 47fcc6b572c37f2e5eeb2f9727848931/a: closing flushed file at 1732710498825Flushing 47fcc6b572c37f2e5eeb2f9727848931/b: creating writer at 1732710498840 (+15 ms)Flushing 47fcc6b572c37f2e5eeb2f9727848931/b: appending metadata at 1732710498860 (+20 ms)Flushing 47fcc6b572c37f2e5eeb2f9727848931/b: closing flushed file at 1732710498860Flushing 47fcc6b572c37f2e5eeb2f9727848931/c: creating writer at 1732710498876 (+16 ms)Flushing 47fcc6b572c37f2e5eeb2f9727848931/c: appending metadata at 1732710498905 (+29 ms)Flushing 47fcc6b572c37f2e5eeb2f9727848931/c: closing flushed file at 1732710498905Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e800d7d: reopening flushed file at 1732710498919 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75a1e131: reopening flushed file at 1732710498926 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6745a093: reopening flushed file at 1732710498934 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 47fcc6b572c37f2e5eeb2f9727848931 in 138ms, sequenceid=6000, compaction requested=false; wal=null at 1732710498942 (+8 ms)Cleaning up temporary data from old regions at 1732710498945 (+3 ms)Region opened successfully at 1732710498953 (+8 ms) 2024-11-27T12:28:18,955 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-27T12:28:18,955 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 47fcc6b572c37f2e5eeb2f9727848931, disabling compactions & flushes 2024-11-27T12:28:18,955 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732710498119.47fcc6b572c37f2e5eeb2f9727848931. 2024-11-27T12:28:18,955 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732710498119.47fcc6b572c37f2e5eeb2f9727848931. 2024-11-27T12:28:18,955 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732710498119.47fcc6b572c37f2e5eeb2f9727848931. after waiting 0 ms 2024-11-27T12:28:18,955 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732710498119.47fcc6b572c37f2e5eeb2f9727848931. 2024-11-27T12:28:18,957 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732710498119.47fcc6b572c37f2e5eeb2f9727848931. 2024-11-27T12:28:18,957 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 47fcc6b572c37f2e5eeb2f9727848931: Waiting for close lock at 1732710498955Disabling compacts and flushes for region at 1732710498955Disabling writes for close at 1732710498955Writing region close event to WAL at 1732710498957 (+2 ms)Closed at 1732710498957 2024-11-27T12:28:18,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741870_1047 (size=93) 2024-11-27T12:28:18,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741870_1047 (size=93) 2024-11-27T12:28:18,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741870_1047 (size=93) 2024-11-27T12:28:18,963 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T12:28:18,963 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732710498673) 2024-11-27T12:28:18,975 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=406 (was 403) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:52554 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:52536 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:39708 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:53254 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=929 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=324 (was 324), ProcessCount=11 (was 11), AvailableMemoryMB=8968 (was 9120) 2024-11-27T12:28:18,985 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=406, OpenFileDescriptor=929, MaxFileDescriptor=1048576, SystemLoadAverage=324, ProcessCount=11, AvailableMemoryMB=8966 2024-11-27T12:28:19,000 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:19,006 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732710498999, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:19,007 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732710499006 2024-11-27T12:28:19,014 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732710498999/wal.1732710499006 2024-11-27T12:28:19,016 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:34427:34427)] 2024-11-27T12:28:19,021 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 268b15094508680cbe9ea75346407440, NAME => 'testSequentialEditLogSeqNum,,1732710499000.268b15094508680cbe9ea75346407440.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:19,021 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732710499000.268b15094508680cbe9ea75346407440.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:19,021 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 268b15094508680cbe9ea75346407440 2024-11-27T12:28:19,021 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 268b15094508680cbe9ea75346407440 2024-11-27T12:28:19,022 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:38485/hbase/data/default/testSequentialEditLogSeqNum/268b15094508680cbe9ea75346407440 doesn't exist for region: 268b15094508680cbe9ea75346407440 on table testSequentialEditLogSeqNum 2024-11-27T12:28:19,023 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 268b15094508680cbe9ea75346407440 on table testSequentialEditLogSeqNum 2024-11-27T12:28:19,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741875_1052 (size=62) 2024-11-27T12:28:19,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741875_1052 (size=62) 2024-11-27T12:28:19,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741875_1052 (size=62) 2024-11-27T12:28:19,039 INFO [StoreOpener-268b15094508680cbe9ea75346407440-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 268b15094508680cbe9ea75346407440 2024-11-27T12:28:19,040 INFO [StoreOpener-268b15094508680cbe9ea75346407440-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 268b15094508680cbe9ea75346407440 columnFamilyName a 2024-11-27T12:28:19,040 DEBUG [StoreOpener-268b15094508680cbe9ea75346407440-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:19,041 INFO [StoreOpener-268b15094508680cbe9ea75346407440-1 {}] regionserver.HStore(327): Store=268b15094508680cbe9ea75346407440/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:19,041 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 268b15094508680cbe9ea75346407440 2024-11-27T12:28:19,041 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testSequentialEditLogSeqNum/268b15094508680cbe9ea75346407440 2024-11-27T12:28:19,042 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testSequentialEditLogSeqNum/268b15094508680cbe9ea75346407440 2024-11-27T12:28:19,042 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 268b15094508680cbe9ea75346407440 2024-11-27T12:28:19,042 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 268b15094508680cbe9ea75346407440 2024-11-27T12:28:19,044 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 268b15094508680cbe9ea75346407440 2024-11-27T12:28:19,047 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testSequentialEditLogSeqNum/268b15094508680cbe9ea75346407440/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:28:19,048 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 268b15094508680cbe9ea75346407440; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61082320, jitterRate=-0.08980250358581543}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T12:28:19,049 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 268b15094508680cbe9ea75346407440: Writing region info on filesystem at 1732710499021Initializing all the Stores at 1732710499038 (+17 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710499038Cleaning up temporary data from old regions at 1732710499042 (+4 ms)Region opened successfully at 1732710499049 (+7 ms) 2024-11-27T12:28:19,060 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-27T12:28:19,061 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-27T12:28:19,062 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 268b15094508680cbe9ea75346407440 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-27T12:28:19,063 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T12:28:19,063 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-27T12:28:19,064 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-27T12:28:19,064 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-27T12:28:19,065 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-27T12:28:19,065 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-27T12:28:19,067 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-27T12:28:19,067 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-27T12:28:19,090 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testSequentialEditLogSeqNum/268b15094508680cbe9ea75346407440/.tmp/a/dda3d3872d70484495969ff4f23298d2 is 81, key is testSequentialEditLogSeqNum/a:x0/1732710499049/Put/seqid=0 2024-11-27T12:28:19,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741876_1053 (size=5833) 2024-11-27T12:28:19,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741876_1053 (size=5833) 2024-11-27T12:28:19,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741876_1053 (size=5833) 2024-11-27T12:28:19,098 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testSequentialEditLogSeqNum/268b15094508680cbe9ea75346407440/.tmp/a/dda3d3872d70484495969ff4f23298d2 2024-11-27T12:28:19,106 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testSequentialEditLogSeqNum/268b15094508680cbe9ea75346407440/.tmp/a/dda3d3872d70484495969ff4f23298d2 as hdfs://localhost:38485/hbase/data/default/testSequentialEditLogSeqNum/268b15094508680cbe9ea75346407440/a/dda3d3872d70484495969ff4f23298d2 2024-11-27T12:28:19,113 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testSequentialEditLogSeqNum/268b15094508680cbe9ea75346407440/a/dda3d3872d70484495969ff4f23298d2, entries=10, sequenceid=13, filesize=5.7 K 2024-11-27T12:28:19,115 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 268b15094508680cbe9ea75346407440 in 52ms, sequenceid=13, compaction requested=false 2024-11-27T12:28:19,115 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 268b15094508680cbe9ea75346407440: 2024-11-27T12:28:19,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T12:28:19,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T12:28:19,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T12:28:19,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T12:28:19,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T12:28:19,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741874_1051 (size=3017) 2024-11-27T12:28:19,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741874_1051 (size=3017) 2024-11-27T12:28:19,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741874_1051 (size=3017) 2024-11-27T12:28:19,143 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38485/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732710498999/wal.1732710499006, size=2.9 K (3017bytes) 2024-11-27T12:28:19,143 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38485/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732710498999/wal.1732710499006 2024-11-27T12:28:19,143 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38485/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732710498999/wal.1732710499006 after 0ms 2024-11-27T12:28:19,146 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38485/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732710498999/wal.1732710499006 took 4ms 2024-11-27T12:28:19,148 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38485/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732710498999/wal.1732710499006 so closing down 2024-11-27T12:28:19,148 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T12:28:19,149 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732710499006.temp 2024-11-27T12:28:19,151 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testSequentialEditLogSeqNum/268b15094508680cbe9ea75346407440/recovered.edits/0000000000000000003-wal.1732710499006.temp 2024-11-27T12:28:19,151 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T12:28:19,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741877_1054 (size=2433) 2024-11-27T12:28:19,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741877_1054 (size=2433) 2024-11-27T12:28:19,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741877_1054 (size=2433) 2024-11-27T12:28:19,160 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testSequentialEditLogSeqNum/268b15094508680cbe9ea75346407440/recovered.edits/0000000000000000003-wal.1732710499006.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-27T12:28:19,161 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38485/hbase/data/default/testSequentialEditLogSeqNum/268b15094508680cbe9ea75346407440/recovered.edits/0000000000000000003-wal.1732710499006.temp to hdfs://localhost:38485/hbase/data/default/testSequentialEditLogSeqNum/268b15094508680cbe9ea75346407440/recovered.edits/0000000000000000020 2024-11-27T12:28:19,161 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:38485/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732710498999/wal.1732710499006, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-11-27T12:28:19,162 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38485/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732710498999/wal.1732710499006, journal: Splitting hdfs://localhost:38485/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732710498999/wal.1732710499006, size=2.9 K (3017bytes) at 1732710499143Finishing writing output for hdfs://localhost:38485/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732710498999/wal.1732710499006 so closing down at 1732710499148 (+5 ms)Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testSequentialEditLogSeqNum/268b15094508680cbe9ea75346407440/recovered.edits/0000000000000000003-wal.1732710499006.temp at 1732710499151 (+3 ms)3 split writer threads finished at 1732710499151Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testSequentialEditLogSeqNum/268b15094508680cbe9ea75346407440/recovered.edits/0000000000000000003-wal.1732710499006.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732710499160 (+9 ms)Rename recovered edits hdfs://localhost:38485/hbase/data/default/testSequentialEditLogSeqNum/268b15094508680cbe9ea75346407440/recovered.edits/0000000000000000003-wal.1732710499006.temp to hdfs://localhost:38485/hbase/data/default/testSequentialEditLogSeqNum/268b15094508680cbe9ea75346407440/recovered.edits/0000000000000000020 at 1732710499161 (+1 ms)Processed 17 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:38485/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732710498999/wal.1732710499006, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1732710499161 2024-11-27T12:28:19,179 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=411 (was 406) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:52554 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:52536 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:53254 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=967 (was 929) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=324 (was 324), ProcessCount=11 (was 11), AvailableMemoryMB=8942 (was 8966) 2024-11-27T12:28:19,194 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=411, OpenFileDescriptor=967, MaxFileDescriptor=1048576, SystemLoadAverage=324, ProcessCount=11, AvailableMemoryMB=8937 2024-11-27T12:28:19,218 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:19,221 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:19,257 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:28:19,260 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-87152595, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/hregion-87152595, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:19,281 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-87152595/hregion-87152595.1732710499261, exclude list is [], retry=0 2024-11-27T12:28:19,284 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:19,285 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:19,285 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:19,287 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-87152595/hregion-87152595.1732710499261 2024-11-27T12:28:19,288 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:46317:46317)] 2024-11-27T12:28:19,288 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 28bdaea3660a28164864a6d28db63f02, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732710499219.28bdaea3660a28164864a6d28db63f02.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38485/hbase 2024-11-27T12:28:19,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741879_1056 (size=70) 2024-11-27T12:28:19,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741879_1056 (size=70) 2024-11-27T12:28:19,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741879_1056 (size=70) 2024-11-27T12:28:19,309 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732710499219.28bdaea3660a28164864a6d28db63f02.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:19,311 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,313 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28bdaea3660a28164864a6d28db63f02 columnFamilyName a 2024-11-27T12:28:19,313 DEBUG [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:19,314 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] regionserver.HStore(327): Store=28bdaea3660a28164864a6d28db63f02/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:19,314 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,316 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28bdaea3660a28164864a6d28db63f02 columnFamilyName b 2024-11-27T12:28:19,316 DEBUG [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:19,317 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] regionserver.HStore(327): Store=28bdaea3660a28164864a6d28db63f02/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:19,317 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,319 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28bdaea3660a28164864a6d28db63f02 columnFamilyName c 2024-11-27T12:28:19,319 DEBUG [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:19,320 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] regionserver.HStore(327): Store=28bdaea3660a28164864a6d28db63f02/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:19,320 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,321 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,322 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,323 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,323 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,324 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T12:28:19,326 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,329 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:28:19,330 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28bdaea3660a28164864a6d28db63f02; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70961715, jitterRate=0.057411953806877136}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T12:28:19,331 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28bdaea3660a28164864a6d28db63f02: Writing region info on filesystem at 1732710499309Initializing all the Stores at 1732710499310 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710499311 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710499311Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710499311Cleaning up temporary data from old regions at 1732710499323 (+12 ms)Region opened successfully at 1732710499331 (+8 ms) 2024-11-27T12:28:19,331 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 28bdaea3660a28164864a6d28db63f02, disabling compactions & flushes 2024-11-27T12:28:19,331 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732710499219.28bdaea3660a28164864a6d28db63f02. 2024-11-27T12:28:19,331 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732710499219.28bdaea3660a28164864a6d28db63f02. 2024-11-27T12:28:19,331 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732710499219.28bdaea3660a28164864a6d28db63f02. after waiting 0 ms 2024-11-27T12:28:19,331 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732710499219.28bdaea3660a28164864a6d28db63f02. 2024-11-27T12:28:19,332 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732710499219.28bdaea3660a28164864a6d28db63f02. 2024-11-27T12:28:19,332 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 28bdaea3660a28164864a6d28db63f02: Waiting for close lock at 1732710499331Disabling compacts and flushes for region at 1732710499331Disabling writes for close at 1732710499331Writing region close event to WAL at 1732710499332 (+1 ms)Closed at 1732710499332 2024-11-27T12:28:19,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741878_1055 (size=93) 2024-11-27T12:28:19,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741878_1055 (size=93) 2024-11-27T12:28:19,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741878_1055 (size=93) 2024-11-27T12:28:19,338 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T12:28:19,339 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-87152595:(num 1732710499261) 2024-11-27T12:28:19,339 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:19,341 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:19,355 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710499341, exclude list is [], retry=0 2024-11-27T12:28:19,359 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:19,359 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:19,359 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:19,361 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710499341 2024-11-27T12:28:19,362 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:46317:46317)] 2024-11-27T12:28:19,362 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 28bdaea3660a28164864a6d28db63f02, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732710499219.28bdaea3660a28164864a6d28db63f02.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:19,362 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732710499219.28bdaea3660a28164864a6d28db63f02.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:19,362 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,362 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,364 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,365 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28bdaea3660a28164864a6d28db63f02 columnFamilyName a 2024-11-27T12:28:19,365 DEBUG [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:19,366 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] regionserver.HStore(327): Store=28bdaea3660a28164864a6d28db63f02/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:19,366 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,367 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28bdaea3660a28164864a6d28db63f02 columnFamilyName b 2024-11-27T12:28:19,367 DEBUG [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:19,368 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] regionserver.HStore(327): Store=28bdaea3660a28164864a6d28db63f02/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:19,368 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,369 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28bdaea3660a28164864a6d28db63f02 columnFamilyName c 2024-11-27T12:28:19,369 DEBUG [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:19,369 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] regionserver.HStore(327): Store=28bdaea3660a28164864a6d28db63f02/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:19,369 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,370 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,371 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,373 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,373 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,373 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T12:28:19,374 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:19,375 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28bdaea3660a28164864a6d28db63f02; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70220792, jitterRate=0.04637134075164795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T12:28:19,376 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28bdaea3660a28164864a6d28db63f02: Writing region info on filesystem at 1732710499362Initializing all the Stores at 1732710499363 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710499364 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710499364Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710499364Cleaning up temporary data from old regions at 1732710499373 (+9 ms)Region opened successfully at 1732710499376 (+3 ms) 2024-11-27T12:28:19,382 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732710499381/Put/seqid=0 2024-11-27T12:28:19,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741881_1058 (size=4826) 2024-11-27T12:28:19,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741881_1058 (size=4826) 2024-11-27T12:28:19,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741881_1058 (size=4826) 2024-11-27T12:28:19,390 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38485/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 28bdaea3660a28164864a6d28db63f02/a 2024-11-27T12:28:19,399 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-27T12:28:19,399 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-27T12:28:19,399 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 28bdaea3660a28164864a6d28db63f02: 2024-11-27T12:28:19,401 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/a/f286236b428944a884c6f32a261980f1_SeqId_3_ 2024-11-27T12:28:19,402 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38485/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 28bdaea3660a28164864a6d28db63f02/a as hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/a/f286236b428944a884c6f32a261980f1_SeqId_3_ - updating store file list. 2024-11-27T12:28:19,408 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for f286236b428944a884c6f32a261980f1_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-27T12:28:19,409 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/a/f286236b428944a884c6f32a261980f1_SeqId_3_ into 28bdaea3660a28164864a6d28db63f02/a 2024-11-27T12:28:19,409 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38485/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 28bdaea3660a28164864a6d28db63f02/a (new location: hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/a/f286236b428944a884c6f32a261980f1_SeqId_3_) 2024-11-27T12:28:19,448 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38485/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710499341, size=0 (0bytes) 2024-11-27T12:28:19,448 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38485/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710499341 might be still open, length is 0 2024-11-27T12:28:19,448 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38485/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710499341 2024-11-27T12:28:19,448 WARN [IPC Server handler 4 on default port 38485 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710499341 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-27T12:28:19,449 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38485/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710499341 after 1ms 2024-11-27T12:28:19,757 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:52772 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:38365:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52772 dst: /127.0.0.1:38365 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38365 remote=/127.0.0.1:52772]. Total timeout mills is 60000, 59659 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:19,758 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:39828 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:38877:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39828 dst: /127.0.0.1:38877 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:19,758 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:53340 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53340 dst: /127.0.0.1:37469 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:19,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741880_1059 (size=504) 2024-11-27T12:28:19,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741880_1059 (size=504) 2024-11-27T12:28:19,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741880_1059 (size=504) 2024-11-27T12:28:23,451 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38485/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710499341 after 4002ms 2024-11-27T12:28:23,455 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38485/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710499341 took 4007ms 2024-11-27T12:28:23,457 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38485/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710499341; continuing. 2024-11-27T12:28:23,457 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38485/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710499341 so closing down 2024-11-27T12:28:23,457 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T12:28:23,459 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732710499341.temp 2024-11-27T12:28:23,461 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/recovered.edits/0000000000000000005-wal.1732710499341.temp 2024-11-27T12:28:23,461 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T12:28:23,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741882_1060 (size=238) 2024-11-27T12:28:23,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741882_1060 (size=238) 2024-11-27T12:28:23,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741882_1060 (size=238) 2024-11-27T12:28:23,470 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/recovered.edits/0000000000000000005-wal.1732710499341.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-27T12:28:23,472 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/recovered.edits/0000000000000000005-wal.1732710499341.temp to hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/recovered.edits/0000000000000000005 2024-11-27T12:28:23,472 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 16 ms; skipped=1; WAL=hdfs://localhost:38485/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710499341, size=0, length=0, corrupted=false, cancelled=false 2024-11-27T12:28:23,472 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38485/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710499341, journal: Splitting hdfs://localhost:38485/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710499341, size=0 (0bytes) at 1732710499448Finishing writing output for hdfs://localhost:38485/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710499341 so closing down at 1732710503457 (+4009 ms)Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/recovered.edits/0000000000000000005-wal.1732710499341.temp at 1732710503461 (+4 ms)3 split writer threads finished at 1732710503461Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/recovered.edits/0000000000000000005-wal.1732710499341.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732710503470 (+9 ms)Rename recovered edits hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/recovered.edits/0000000000000000005-wal.1732710499341.temp to hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/recovered.edits/0000000000000000005 at 1732710503472 (+2 ms)Processed 2 edits across 1 Regions in 16 ms; skipped=1; WAL=hdfs://localhost:38485/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710499341, size=0, length=0, corrupted=false, cancelled=false at 1732710503472 2024-11-27T12:28:23,474 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38485/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710499341 to hdfs://localhost:38485/hbase/oldWALs/wal.1732710499341 2024-11-27T12:28:23,475 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/recovered.edits/0000000000000000005 2024-11-27T12:28:23,475 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:23,477 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:23,491 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710503477, exclude list is [], retry=0 2024-11-27T12:28:23,494 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:23,494 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:23,495 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:23,497 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710503477 2024-11-27T12:28:23,497 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:34427:34427)] 2024-11-27T12:28:23,497 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 28bdaea3660a28164864a6d28db63f02, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732710499219.28bdaea3660a28164864a6d28db63f02.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:23,497 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732710499219.28bdaea3660a28164864a6d28db63f02.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:23,498 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:23,498 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:23,499 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:23,500 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28bdaea3660a28164864a6d28db63f02 columnFamilyName a 2024-11-27T12:28:23,500 DEBUG [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:23,509 DEBUG [StoreFileOpener-28bdaea3660a28164864a6d28db63f02-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for f286236b428944a884c6f32a261980f1_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-27T12:28:23,509 DEBUG [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/a/f286236b428944a884c6f32a261980f1_SeqId_3_ 2024-11-27T12:28:23,509 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] regionserver.HStore(327): Store=28bdaea3660a28164864a6d28db63f02/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:23,509 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:23,511 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28bdaea3660a28164864a6d28db63f02 columnFamilyName b 2024-11-27T12:28:23,511 DEBUG [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:23,511 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] regionserver.HStore(327): Store=28bdaea3660a28164864a6d28db63f02/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:23,511 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:23,513 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28bdaea3660a28164864a6d28db63f02 columnFamilyName c 2024-11-27T12:28:23,513 DEBUG [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:23,513 INFO [StoreOpener-28bdaea3660a28164864a6d28db63f02-1 {}] regionserver.HStore(327): Store=28bdaea3660a28164864a6d28db63f02/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:23,513 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:23,514 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:23,516 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:23,516 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/recovered.edits/0000000000000000005 2024-11-27T12:28:23,520 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/recovered.edits/0000000000000000005 2024-11-27T12:28:23,520 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 28bdaea3660a28164864a6d28db63f02 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-27T12:28:23,535 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/.tmp/a/d006443795d84fafafe74663f10bfdf3 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732710499414/Put/seqid=0 2024-11-27T12:28:23,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741884_1062 (size=5149) 2024-11-27T12:28:23,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741884_1062 (size=5149) 2024-11-27T12:28:23,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741884_1062 (size=5149) 2024-11-27T12:28:23,543 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/.tmp/a/d006443795d84fafafe74663f10bfdf3 2024-11-27T12:28:23,551 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/.tmp/a/d006443795d84fafafe74663f10bfdf3 as hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/a/d006443795d84fafafe74663f10bfdf3 2024-11-27T12:28:23,557 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/a/d006443795d84fafafe74663f10bfdf3, entries=1, sequenceid=5, filesize=5.0 K 2024-11-27T12:28:23,557 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 28bdaea3660a28164864a6d28db63f02 in 37ms, sequenceid=5, compaction requested=false; wal=null 2024-11-27T12:28:23,558 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/recovered.edits/0000000000000000005 2024-11-27T12:28:23,559 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:23,560 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:23,560 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T12:28:23,562 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28bdaea3660a28164864a6d28db63f02 2024-11-27T12:28:23,564 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/28bdaea3660a28164864a6d28db63f02/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-27T12:28:23,565 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28bdaea3660a28164864a6d28db63f02; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62797858, jitterRate=-0.06423899531364441}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T12:28:23,566 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28bdaea3660a28164864a6d28db63f02: Writing region info on filesystem at 1732710503498Initializing all the Stores at 1732710503499 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710503499Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710503499Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710503499Obtaining lock to block concurrent updates at 1732710503520 (+21 ms)Preparing flush snapshotting stores in 28bdaea3660a28164864a6d28db63f02 at 1732710503520Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732710499219.28bdaea3660a28164864a6d28db63f02., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732710503520Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732710499219.28bdaea3660a28164864a6d28db63f02. at 1732710503520Flushing 28bdaea3660a28164864a6d28db63f02/a: creating writer at 1732710503520Flushing 28bdaea3660a28164864a6d28db63f02/a: appending metadata at 1732710503534 (+14 ms)Flushing 28bdaea3660a28164864a6d28db63f02/a: closing flushed file at 1732710503534Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a9b60b8: reopening flushed file at 1732710503549 (+15 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 28bdaea3660a28164864a6d28db63f02 in 37ms, sequenceid=5, compaction requested=false; wal=null at 1732710503557 (+8 ms)Cleaning up temporary data from old regions at 1732710503560 (+3 ms)Region opened successfully at 1732710503566 (+6 ms) 2024-11-27T12:28:23,570 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 28bdaea3660a28164864a6d28db63f02, disabling compactions & flushes 2024-11-27T12:28:23,570 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732710499219.28bdaea3660a28164864a6d28db63f02. 2024-11-27T12:28:23,570 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732710499219.28bdaea3660a28164864a6d28db63f02. 2024-11-27T12:28:23,570 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732710499219.28bdaea3660a28164864a6d28db63f02. after waiting 0 ms 2024-11-27T12:28:23,570 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732710499219.28bdaea3660a28164864a6d28db63f02. 2024-11-27T12:28:23,571 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732710499219.28bdaea3660a28164864a6d28db63f02. 2024-11-27T12:28:23,572 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 28bdaea3660a28164864a6d28db63f02: Waiting for close lock at 1732710503570Disabling compacts and flushes for region at 1732710503570Disabling writes for close at 1732710503570Writing region close event to WAL at 1732710503571 (+1 ms)Closed at 1732710503571 2024-11-27T12:28:23,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741883_1061 (size=93) 2024-11-27T12:28:23,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741883_1061 (size=93) 2024-11-27T12:28:23,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741883_1061 (size=93) 2024-11-27T12:28:23,577 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T12:28:23,577 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732710503477) 2024-11-27T12:28:23,589 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=419 (was 411) Potentially hanging thread: IPC Client (1612229954) connection to localhost/127.0.0.1:39259 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-606390546_22 at /127.0.0.1:47840 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1612229954) connection to localhost/127.0.0.1:38485 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-606390546_22 at /127.0.0.1:41916 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39259 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:38485 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-606390546_22 at /127.0.0.1:56430 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1027 (was 967) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=298 (was 324), ProcessCount=11 (was 11), AvailableMemoryMB=8886 (was 8937) 2024-11-27T12:28:23,590 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1027 is superior to 1024 2024-11-27T12:28:23,601 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=419, OpenFileDescriptor=1027, MaxFileDescriptor=1048576, SystemLoadAverage=298, ProcessCount=11, AvailableMemoryMB=8884 2024-11-27T12:28:23,601 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1027 is superior to 1024 2024-11-27T12:28:23,617 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:23,622 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T12:28:23,626 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 1ba5af43fd99,46311,1732710488723 2024-11-27T12:28:23,629 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@25f1137e 2024-11-27T12:28:23,629 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T12:28:23,632 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T12:28:23,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46311 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T12:28:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46311 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-27T12:28:23,644 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T12:28:23,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46311 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-27T12:28:23,647 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:23,648 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T12:28:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46311 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T12:28:23,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741885_1063 (size=694) 2024-11-27T12:28:23,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741885_1063 (size=694) 2024-11-27T12:28:23,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741885_1063 (size=694) 2024-11-27T12:28:23,668 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5556f2e573f63651ecac0666fe4456cb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606 2024-11-27T12:28:23,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741886_1064 (size=77) 2024-11-27T12:28:23,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741886_1064 (size=77) 2024-11-27T12:28:23,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741886_1064 (size=77) 2024-11-27T12:28:23,678 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:23,678 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 5556f2e573f63651ecac0666fe4456cb, disabling compactions & flushes 2024-11-27T12:28:23,678 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:23,678 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:23,678 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. after waiting 0 ms 2024-11-27T12:28:23,678 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:23,678 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:23,678 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5556f2e573f63651ecac0666fe4456cb: Waiting for close lock at 1732710503678Disabling compacts and flushes for region at 1732710503678Disabling writes for close at 1732710503678Writing region close event to WAL at 1732710503678Closed at 1732710503678 2024-11-27T12:28:23,680 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T12:28:23,684 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732710503680"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732710503680"}]},"ts":"1732710503680"} 2024-11-27T12:28:23,688 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T12:28:23,690 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T12:28:23,692 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710503690"}]},"ts":"1732710503690"} 2024-11-27T12:28:23,696 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-27T12:28:23,697 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {1ba5af43fd99=0} racks are {/default-rack=0} 2024-11-27T12:28:23,699 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T12:28:23,699 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T12:28:23,699 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-27T12:28:23,699 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T12:28:23,699 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T12:28:23,699 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-27T12:28:23,699 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T12:28:23,699 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T12:28:23,699 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-27T12:28:23,699 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T12:28:23,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, ASSIGN}] 2024-11-27T12:28:23,703 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, ASSIGN 2024-11-27T12:28:23,704 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, ASSIGN; state=OFFLINE, location=1ba5af43fd99,35527,1732710489490; forceNewPlan=false, retain=false 2024-11-27T12:28:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46311 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T12:28:23,857 INFO [1ba5af43fd99:46311 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-27T12:28:23,858 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5556f2e573f63651ecac0666fe4456cb, regionState=OPENING, regionLocation=1ba5af43fd99,35527,1732710489490 2024-11-27T12:28:23,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, ASSIGN because future has completed 2024-11-27T12:28:23,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5556f2e573f63651ecac0666fe4456cb, server=1ba5af43fd99,35527,1732710489490}] 2024-11-27T12:28:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46311 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T12:28:24,029 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:24,029 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5556f2e573f63651ecac0666fe4456cb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:24,030 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,030 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:24,030 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,030 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,033 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,036 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5556f2e573f63651ecac0666fe4456cb columnFamilyName cf1 2024-11-27T12:28:24,036 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:24,037 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(327): Store=5556f2e573f63651ecac0666fe4456cb/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:24,037 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,041 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5556f2e573f63651ecac0666fe4456cb columnFamilyName cf2 2024-11-27T12:28:24,041 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:24,042 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(327): Store=5556f2e573f63651ecac0666fe4456cb/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:24,042 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,043 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,044 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,044 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,044 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,045 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-27T12:28:24,047 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,049 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:28:24,049 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5556f2e573f63651ecac0666fe4456cb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69924202, jitterRate=0.04195180535316467}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-27T12:28:24,050 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,070 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5556f2e573f63651ecac0666fe4456cb: Running coprocessor pre-open hook at 1732710504031Writing region info on filesystem at 1732710504031Initializing all the Stores at 1732710504032 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710504032Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710504033 (+1 ms)Cleaning up temporary data from old regions at 1732710504044 (+11 ms)Running coprocessor post-open hooks at 1732710504050 (+6 ms)Region opened successfully at 1732710504070 (+20 ms) 2024-11-27T12:28:24,072 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., pid=6, masterSystemTime=1732710504019 2024-11-27T12:28:24,075 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:24,075 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:24,076 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5556f2e573f63651ecac0666fe4456cb, regionState=OPEN, openSeqNum=2, regionLocation=1ba5af43fd99,35527,1732710489490 2024-11-27T12:28:24,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5556f2e573f63651ecac0666fe4456cb, server=1ba5af43fd99,35527,1732710489490 because future has completed 2024-11-27T12:28:24,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-27T12:28:24,085 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5556f2e573f63651ecac0666fe4456cb, server=1ba5af43fd99,35527,1732710489490 in 217 msec 2024-11-27T12:28:24,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-27T12:28:24,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, ASSIGN in 383 msec 2024-11-27T12:28:24,088 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T12:28:24,089 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710504088"}]},"ts":"1732710504088"} 2024-11-27T12:28:24,091 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-27T12:28:24,092 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T12:28:24,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 456 msec 2024-11-27T12:28:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46311 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T12:28:24,285 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-27T12:28:24,285 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-27T12:28:24,287 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T12:28:24,292 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-27T12:28:24,292 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T12:28:24,293 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-27T12:28:24,304 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,35527,1732710489490, seqNum=2] 2024-11-27T12:28:24,323 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46311 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=5556f2e573f63651ecac0666fe4456cb, source=1ba5af43fd99,35527,1732710489490, destination=1ba5af43fd99,41787,1732710489587, warming up region on 1ba5af43fd99,41787,1732710489587 2024-11-27T12:28:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46311 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T12:28:24,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46311 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=5556f2e573f63651ecac0666fe4456cb, source=1ba5af43fd99,35527,1732710489490, destination=1ba5af43fd99,41787,1732710489587, running balancer 2024-11-27T12:28:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46311 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, REOPEN/MOVE 2024-11-27T12:28:24,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41213, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T12:28:24,327 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, REOPEN/MOVE 2024-11-27T12:28:24,330 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5556f2e573f63651ecac0666fe4456cb, regionState=CLOSING, regionLocation=1ba5af43fd99,35527,1732710489490 2024-11-27T12:28:24,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41787 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41787 {}] regionserver.HRegion(7855): Warmup {ENCODED => 5556f2e573f63651ecac0666fe4456cb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41787 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:24,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, REOPEN/MOVE because future has completed 2024-11-27T12:28:24,334 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T12:28:24,334 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5556f2e573f63651ecac0666fe4456cb, server=1ba5af43fd99,35527,1732710489490}] 2024-11-27T12:28:24,335 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5556f2e573f63651ecac0666fe4456cb columnFamilyName cf1 2024-11-27T12:28:24,335 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:24,336 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(327): Store=5556f2e573f63651ecac0666fe4456cb/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:24,336 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,337 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5556f2e573f63651ecac0666fe4456cb columnFamilyName cf2 2024-11-27T12:28:24,337 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:24,338 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(327): Store=5556f2e573f63651ecac0666fe4456cb/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41787 {}] regionserver.HRegion(1722): Closing 5556f2e573f63651ecac0666fe4456cb, disabling compactions & flushes 2024-11-27T12:28:24,338 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41787 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41787 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41787 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. after waiting 0 ms 2024-11-27T12:28:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41787 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:24,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41787 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41787 {}] regionserver.HRegion(1676): Region close journal for 5556f2e573f63651ecac0666fe4456cb: Waiting for close lock at 1732710504338Disabling compacts and flushes for region at 1732710504338Disabling writes for close at 1732710504338Writing region close event to WAL at 1732710504343 (+5 ms)Closed at 1732710504343 2024-11-27T12:28:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46311 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-27T12:28:24,496 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,496 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T12:28:24,497 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 5556f2e573f63651ecac0666fe4456cb, disabling compactions & flushes 2024-11-27T12:28:24,497 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:24,497 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:24,498 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. after waiting 0 ms 2024-11-27T12:28:24,498 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:24,498 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 5556f2e573f63651ecac0666fe4456cb 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-27T12:28:24,519 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/.tmp/cf1/4ddcdc4c8c154260a4ec956293caf748 is 35, key is r1/cf1:q/1732710504306/Put/seqid=0 2024-11-27T12:28:24,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741887_1065 (size=4783) 2024-11-27T12:28:24,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741887_1065 (size=4783) 2024-11-27T12:28:24,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741887_1065 (size=4783) 2024-11-27T12:28:24,526 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/.tmp/cf1/4ddcdc4c8c154260a4ec956293caf748 2024-11-27T12:28:24,534 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/.tmp/cf1/4ddcdc4c8c154260a4ec956293caf748 as hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf1/4ddcdc4c8c154260a4ec956293caf748 2024-11-27T12:28:24,539 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf1/4ddcdc4c8c154260a4ec956293caf748, entries=1, sequenceid=5, filesize=4.7 K 2024-11-27T12:28:24,540 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 5556f2e573f63651ecac0666fe4456cb in 42ms, sequenceid=5, compaction requested=false 2024-11-27T12:28:24,541 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-27T12:28:24,546 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-27T12:28:24,548 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:24,548 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 5556f2e573f63651ecac0666fe4456cb: Waiting for close lock at 1732710504497Running coprocessor pre-close hooks at 1732710504497Disabling compacts and flushes for region at 1732710504497Disabling writes for close at 1732710504498 (+1 ms)Obtaining lock to block concurrent updates at 1732710504498Preparing flush snapshotting stores in 5556f2e573f63651ecac0666fe4456cb at 1732710504498Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732710504499 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. at 1732710504500 (+1 ms)Flushing 5556f2e573f63651ecac0666fe4456cb/cf1: creating writer at 1732710504500Flushing 5556f2e573f63651ecac0666fe4456cb/cf1: appending metadata at 1732710504519 (+19 ms)Flushing 5556f2e573f63651ecac0666fe4456cb/cf1: closing flushed file at 1732710504519Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c16b475: reopening flushed file at 1732710504533 (+14 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 5556f2e573f63651ecac0666fe4456cb in 42ms, sequenceid=5, compaction requested=false at 1732710504540 (+7 ms)Writing region close event to WAL at 1732710504542 (+2 ms)Running coprocessor post-close hooks at 1732710504546 (+4 ms)Closed at 1732710504548 (+2 ms) 2024-11-27T12:28:24,549 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 5556f2e573f63651ecac0666fe4456cb move to 1ba5af43fd99,41787,1732710489587 record at close sequenceid=5 2024-11-27T12:28:24,552 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,553 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5556f2e573f63651ecac0666fe4456cb, regionState=CLOSED 2024-11-27T12:28:24,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5556f2e573f63651ecac0666fe4456cb, server=1ba5af43fd99,35527,1732710489490 because future has completed 2024-11-27T12:28:24,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-27T12:28:24,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 5556f2e573f63651ecac0666fe4456cb, server=1ba5af43fd99,35527,1732710489490 in 223 msec 2024-11-27T12:28:24,562 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, REOPEN/MOVE; state=CLOSED, location=1ba5af43fd99,41787,1732710489587; forceNewPlan=false, retain=false 2024-11-27T12:28:24,713 INFO [1ba5af43fd99:46311 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-27T12:28:24,713 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5556f2e573f63651ecac0666fe4456cb, regionState=OPENING, regionLocation=1ba5af43fd99,41787,1732710489587 2024-11-27T12:28:24,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, REOPEN/MOVE because future has completed 2024-11-27T12:28:24,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5556f2e573f63651ecac0666fe4456cb, server=1ba5af43fd99,41787,1732710489587}] 2024-11-27T12:28:24,879 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:24,879 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 5556f2e573f63651ecac0666fe4456cb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:24,880 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,880 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:24,880 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,880 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,882 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,883 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5556f2e573f63651ecac0666fe4456cb columnFamilyName cf1 2024-11-27T12:28:24,883 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:24,890 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf1/4ddcdc4c8c154260a4ec956293caf748 2024-11-27T12:28:24,890 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(327): Store=5556f2e573f63651ecac0666fe4456cb/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:24,890 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,891 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5556f2e573f63651ecac0666fe4456cb columnFamilyName cf2 2024-11-27T12:28:24,891 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:24,892 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(327): Store=5556f2e573f63651ecac0666fe4456cb/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:24,892 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,893 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,894 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,895 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,895 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,896 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-27T12:28:24,897 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,898 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 5556f2e573f63651ecac0666fe4456cb; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69259418, jitterRate=0.03204575181007385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-27T12:28:24,898 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:24,899 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 5556f2e573f63651ecac0666fe4456cb: Running coprocessor pre-open hook at 1732710504880Writing region info on filesystem at 1732710504880Initializing all the Stores at 1732710504881 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710504881Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710504882 (+1 ms)Cleaning up temporary data from old regions at 1732710504895 (+13 ms)Running coprocessor post-open hooks at 1732710504898 (+3 ms)Region opened successfully at 1732710504899 (+1 ms) 2024-11-27T12:28:24,900 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., pid=9, masterSystemTime=1732710504875 2024-11-27T12:28:24,902 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:24,902 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:24,903 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5556f2e573f63651ecac0666fe4456cb, regionState=OPEN, openSeqNum=9, regionLocation=1ba5af43fd99,41787,1732710489587 2024-11-27T12:28:24,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5556f2e573f63651ecac0666fe4456cb, server=1ba5af43fd99,41787,1732710489587 because future has completed 2024-11-27T12:28:24,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-27T12:28:24,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 5556f2e573f63651ecac0666fe4456cb, server=1ba5af43fd99,41787,1732710489587 in 186 msec 2024-11-27T12:28:24,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, REOPEN/MOVE in 584 msec 2024-11-27T12:28:24,935 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T12:28:24,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T12:28:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35527 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:35442 deadline: 1732710564941, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ba5af43fd99 port=41787 startCode=1732710489587. As of locationSeqNum=5. 2024-11-27T12:28:24,948 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,35527,1732710489490, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,35527,1732710489490, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ba5af43fd99 port=41787 startCode=1732710489587. As of locationSeqNum=5. 2024-11-27T12:28:24,949 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,35527,1732710489490, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ba5af43fd99 port=41787 startCode=1732710489587. As of locationSeqNum=5. 2024-11-27T12:28:24,949 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,35527,1732710489490, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,41787,1732710489587, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ba5af43fd99 port=41787 startCode=1732710489587. As of locationSeqNum=5. 2024-11-27T12:28:25,054 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:28:25,056 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:28:25,067 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5556f2e573f63651ecac0666fe4456cb 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-27T12:28:25,090 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/.tmp/cf1/f5f1ade12cd943ef8a06a9106015f017 is 29, key is r1/cf1:/1732710505058/DeleteFamily/seqid=0 2024-11-27T12:28:25,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741888_1066 (size=4906) 2024-11-27T12:28:25,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741888_1066 (size=4906) 2024-11-27T12:28:25,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741888_1066 (size=4906) 2024-11-27T12:28:25,099 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/.tmp/cf1/f5f1ade12cd943ef8a06a9106015f017 2024-11-27T12:28:25,106 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f5f1ade12cd943ef8a06a9106015f017 2024-11-27T12:28:25,122 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/.tmp/cf2/06bb9e68559f4e1f9ea8e8b6479492bf is 29, key is r1/cf2:/1732710505058/DeleteFamily/seqid=0 2024-11-27T12:28:25,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741889_1067 (size=4906) 2024-11-27T12:28:25,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741889_1067 (size=4906) 2024-11-27T12:28:25,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741889_1067 (size=4906) 2024-11-27T12:28:25,132 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/.tmp/cf2/06bb9e68559f4e1f9ea8e8b6479492bf 2024-11-27T12:28:25,140 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 06bb9e68559f4e1f9ea8e8b6479492bf 2024-11-27T12:28:25,141 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/.tmp/cf1/f5f1ade12cd943ef8a06a9106015f017 as hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf1/f5f1ade12cd943ef8a06a9106015f017 2024-11-27T12:28:25,150 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f5f1ade12cd943ef8a06a9106015f017 2024-11-27T12:28:25,151 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf1/f5f1ade12cd943ef8a06a9106015f017, entries=1, sequenceid=12, filesize=4.8 K 2024-11-27T12:28:25,152 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/.tmp/cf2/06bb9e68559f4e1f9ea8e8b6479492bf as hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf2/06bb9e68559f4e1f9ea8e8b6479492bf 2024-11-27T12:28:25,160 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 06bb9e68559f4e1f9ea8e8b6479492bf 2024-11-27T12:28:25,160 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf2/06bb9e68559f4e1f9ea8e8b6479492bf, entries=1, sequenceid=12, filesize=4.8 K 2024-11-27T12:28:25,161 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 5556f2e573f63651ecac0666fe4456cb in 94ms, sequenceid=12, compaction requested=false 2024-11-27T12:28:25,161 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5556f2e573f63651ecac0666fe4456cb: 2024-11-27T12:28:25,164 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T12:28:25,166 DEBUG [Time-limited test {}] regionserver.HStore(1541): 5556f2e573f63651ecac0666fe4456cb/cf1 is initiating major compaction (all files) 2024-11-27T12:28:25,167 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T12:28:25,167 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:25,167 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 5556f2e573f63651ecac0666fe4456cb/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:25,168 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf1/4ddcdc4c8c154260a4ec956293caf748, hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf1/f5f1ade12cd943ef8a06a9106015f017] into tmpdir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/.tmp, totalSize=9.5 K 2024-11-27T12:28:25,169 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4ddcdc4c8c154260a4ec956293caf748, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732710504306 2024-11-27T12:28:25,170 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f5f1ade12cd943ef8a06a9106015f017, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-27T12:28:25,185 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 5556f2e573f63651ecac0666fe4456cb#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:28:25,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741890_1068 (size=4626) 2024-11-27T12:28:25,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741890_1068 (size=4626) 2024-11-27T12:28:25,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741890_1068 (size=4626) 2024-11-27T12:28:25,203 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/.tmp/cf1/35bfd4236e9d440bb96762e929a69bc4 as hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf1/35bfd4236e9d440bb96762e929a69bc4 2024-11-27T12:28:25,216 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 5556f2e573f63651ecac0666fe4456cb/cf1 of 5556f2e573f63651ecac0666fe4456cb into 35bfd4236e9d440bb96762e929a69bc4(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:28:25,217 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 5556f2e573f63651ecac0666fe4456cb: 2024-11-27T12:28:25,217 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-27T12:28:25,217 DEBUG [Time-limited test {}] regionserver.HStore(1541): 5556f2e573f63651ecac0666fe4456cb/cf2 is initiating major compaction (all files) 2024-11-27T12:28:25,217 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T12:28:25,217 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:28:25,217 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 5556f2e573f63651ecac0666fe4456cb/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:25,217 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf2/06bb9e68559f4e1f9ea8e8b6479492bf] into tmpdir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/.tmp, totalSize=4.8 K 2024-11-27T12:28:25,218 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 06bb9e68559f4e1f9ea8e8b6479492bf, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-27T12:28:25,223 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 5556f2e573f63651ecac0666fe4456cb#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:28:25,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741891_1069 (size=4592) 2024-11-27T12:28:25,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741891_1069 (size=4592) 2024-11-27T12:28:25,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741891_1069 (size=4592) 2024-11-27T12:28:25,238 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/.tmp/cf2/f8b52a6c9b3e4d6585fad73fbd1821cc as hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf2/f8b52a6c9b3e4d6585fad73fbd1821cc 2024-11-27T12:28:25,246 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 5556f2e573f63651ecac0666fe4456cb/cf2 of 5556f2e573f63651ecac0666fe4456cb into f8b52a6c9b3e4d6585fad73fbd1821cc(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:28:25,246 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 5556f2e573f63651ecac0666fe4456cb: 2024-11-27T12:28:25,252 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46311 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=5556f2e573f63651ecac0666fe4456cb, source=1ba5af43fd99,41787,1732710489587, destination=1ba5af43fd99,35527,1732710489490, warming up region on 1ba5af43fd99,35527,1732710489490 2024-11-27T12:28:25,252 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46311 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=5556f2e573f63651ecac0666fe4456cb, source=1ba5af43fd99,41787,1732710489587, destination=1ba5af43fd99,35527,1732710489490, running balancer 2024-11-27T12:28:25,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46311 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, REOPEN/MOVE 2024-11-27T12:28:25,254 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, REOPEN/MOVE 2024-11-27T12:28:25,256 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5556f2e573f63651ecac0666fe4456cb, regionState=CLOSING, regionLocation=1ba5af43fd99,41787,1732710489587 2024-11-27T12:28:25,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35527 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35527 {}] regionserver.HRegion(7855): Warmup {ENCODED => 5556f2e573f63651ecac0666fe4456cb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:25,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35527 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:25,257 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:25,258 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5556f2e573f63651ecac0666fe4456cb columnFamilyName cf1 2024-11-27T12:28:25,258 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:25,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, REOPEN/MOVE because future has completed 2024-11-27T12:28:25,260 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T12:28:25,260 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5556f2e573f63651ecac0666fe4456cb, server=1ba5af43fd99,41787,1732710489587}] 2024-11-27T12:28:25,268 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf1/35bfd4236e9d440bb96762e929a69bc4 2024-11-27T12:28:25,274 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf1/4ddcdc4c8c154260a4ec956293caf748 2024-11-27T12:28:25,281 INFO [StoreFileOpener-5556f2e573f63651ecac0666fe4456cb-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f5f1ade12cd943ef8a06a9106015f017 2024-11-27T12:28:25,281 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf1/f5f1ade12cd943ef8a06a9106015f017 2024-11-27T12:28:25,281 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(327): Store=5556f2e573f63651ecac0666fe4456cb/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:25,282 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:25,283 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5556f2e573f63651ecac0666fe4456cb columnFamilyName cf2 2024-11-27T12:28:25,283 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:25,290 INFO [StoreFileOpener-5556f2e573f63651ecac0666fe4456cb-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 06bb9e68559f4e1f9ea8e8b6479492bf 2024-11-27T12:28:25,291 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf2/06bb9e68559f4e1f9ea8e8b6479492bf 2024-11-27T12:28:25,296 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf2/f8b52a6c9b3e4d6585fad73fbd1821cc 2024-11-27T12:28:25,297 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(327): Store=5556f2e573f63651ecac0666fe4456cb/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35527 {}] regionserver.HRegion(1722): Closing 5556f2e573f63651ecac0666fe4456cb, disabling compactions & flushes 2024-11-27T12:28:25,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35527 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35527 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35527 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. after waiting 0 ms 2024-11-27T12:28:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35527 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:25,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35527 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:25,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35527 {}] regionserver.HRegion(1676): Region close journal for 5556f2e573f63651ecac0666fe4456cb: Waiting for close lock at 1732710505297Disabling compacts and flushes for region at 1732710505297Disabling writes for close at 1732710505297Writing region close event to WAL at 1732710505298 (+1 ms)Closed at 1732710505298 2024-11-27T12:28:25,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46311 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-27T12:28:25,414 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:25,414 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T12:28:25,414 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 5556f2e573f63651ecac0666fe4456cb, disabling compactions & flushes 2024-11-27T12:28:25,414 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:25,414 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:25,415 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. after waiting 0 ms 2024-11-27T12:28:25,415 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:25,415 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf1/4ddcdc4c8c154260a4ec956293caf748, hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf1/f5f1ade12cd943ef8a06a9106015f017] to archive 2024-11-27T12:28:25,418 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:28:25,421 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf1/4ddcdc4c8c154260a4ec956293caf748 to hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf1/4ddcdc4c8c154260a4ec956293caf748 2024-11-27T12:28:25,422 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf1/f5f1ade12cd943ef8a06a9106015f017 to hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf1/f5f1ade12cd943ef8a06a9106015f017 2024-11-27T12:28:25,434 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf2/06bb9e68559f4e1f9ea8e8b6479492bf] to archive 2024-11-27T12:28:25,435 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:28:25,437 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf2/06bb9e68559f4e1f9ea8e8b6479492bf to hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf2/06bb9e68559f4e1f9ea8e8b6479492bf 2024-11-27T12:28:25,442 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-27T12:28:25,443 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:25,443 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 5556f2e573f63651ecac0666fe4456cb: Waiting for close lock at 1732710505414Running coprocessor pre-close hooks at 1732710505414Disabling compacts and flushes for region at 1732710505414Disabling writes for close at 1732710505415 (+1 ms)Writing region close event to WAL at 1732710505438 (+23 ms)Running coprocessor post-close hooks at 1732710505443 (+5 ms)Closed at 1732710505443 2024-11-27T12:28:25,443 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 5556f2e573f63651ecac0666fe4456cb move to 1ba5af43fd99,35527,1732710489490 record at close sequenceid=12 2024-11-27T12:28:25,446 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:25,447 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5556f2e573f63651ecac0666fe4456cb, regionState=CLOSED 2024-11-27T12:28:25,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5556f2e573f63651ecac0666fe4456cb, server=1ba5af43fd99,41787,1732710489587 because future has completed 2024-11-27T12:28:25,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-27T12:28:25,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 5556f2e573f63651ecac0666fe4456cb, server=1ba5af43fd99,41787,1732710489587 in 192 msec 2024-11-27T12:28:25,456 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, REOPEN/MOVE; state=CLOSED, location=1ba5af43fd99,35527,1732710489490; forceNewPlan=false, retain=false 2024-11-27T12:28:25,606 INFO [1ba5af43fd99:46311 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-27T12:28:25,607 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5556f2e573f63651ecac0666fe4456cb, regionState=OPENING, regionLocation=1ba5af43fd99,35527,1732710489490 2024-11-27T12:28:25,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, REOPEN/MOVE because future has completed 2024-11-27T12:28:25,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5556f2e573f63651ecac0666fe4456cb, server=1ba5af43fd99,35527,1732710489490}] 2024-11-27T12:28:25,769 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:25,769 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 5556f2e573f63651ecac0666fe4456cb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:25,770 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:25,770 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:25,770 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:25,770 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:25,771 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:25,772 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5556f2e573f63651ecac0666fe4456cb columnFamilyName cf1 2024-11-27T12:28:25,772 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:25,779 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf1/35bfd4236e9d440bb96762e929a69bc4 2024-11-27T12:28:25,780 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(327): Store=5556f2e573f63651ecac0666fe4456cb/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:25,780 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:25,781 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5556f2e573f63651ecac0666fe4456cb columnFamilyName cf2 2024-11-27T12:28:25,781 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:25,788 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf2/f8b52a6c9b3e4d6585fad73fbd1821cc 2024-11-27T12:28:25,788 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(327): Store=5556f2e573f63651ecac0666fe4456cb/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:25,788 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:25,789 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:25,790 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:25,791 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:25,791 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:25,792 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-27T12:28:25,793 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:25,794 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 5556f2e573f63651ecac0666fe4456cb; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72501604, jitterRate=0.08035808801651001}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-27T12:28:25,794 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:25,795 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 5556f2e573f63651ecac0666fe4456cb: Running coprocessor pre-open hook at 1732710505770Writing region info on filesystem at 1732710505770Initializing all the Stores at 1732710505771 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710505771Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710505771Cleaning up temporary data from old regions at 1732710505791 (+20 ms)Running coprocessor post-open hooks at 1732710505794 (+3 ms)Region opened successfully at 1732710505795 (+1 ms) 2024-11-27T12:28:25,795 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., pid=12, masterSystemTime=1732710505766 2024-11-27T12:28:25,798 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:25,798 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:25,798 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5556f2e573f63651ecac0666fe4456cb, regionState=OPEN, openSeqNum=18, regionLocation=1ba5af43fd99,35527,1732710489490 2024-11-27T12:28:25,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5556f2e573f63651ecac0666fe4456cb, server=1ba5af43fd99,35527,1732710489490 because future has completed 2024-11-27T12:28:25,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-27T12:28:25,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 5556f2e573f63651ecac0666fe4456cb, server=1ba5af43fd99,35527,1732710489490 in 189 msec 2024-11-27T12:28:25,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, REOPEN/MOVE in 551 msec 2024-11-27T12:28:25,856 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T12:28:25,859 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T12:28:25,862 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 1ba5af43fd99,35527,1732710489490: testing ***** 2024-11-27T12:28:25,862 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-27T12:28:25,866 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-27T12:28:25,868 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-27T12:28:25,871 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-27T12:28:25,872 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-27T12:28:25,881 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 241703920 }, "NonHeapMemoryUsage": { "committed": 169869312, "init": 7667712, "max": -1, "used": 167259616 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "1ba5af43fd99", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 9, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 8, "TotalCallTime_95th_percentile": 8, "TotalCallTime_98th_percentile": 8, "TotalCallTime_99th_percentile": 8, "TotalCallTime_99.9th_percentile": 8, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "1ba5af43fd99", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:58880", "tag.serverName": "1ba5af43fd99,44849,1732710489640", "tag.clusterId": "7647a787-0323-46fc-9c11-7d9c2370932f", "tag.Context": "regionserver", "tag.Hostname": "1ba5af43fd99", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732710489640, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-27T12:28:25,884 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46311 {}] master.MasterRpcServices(700): 1ba5af43fd99,35527,1732710489490 reported a fatal error: ***** ABORTING region server 1ba5af43fd99,35527,1732710489490: testing ***** 2024-11-27T12:28:25,886 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1ba5af43fd99,35527,1732710489490' ***** 2024-11-27T12:28:25,886 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-27T12:28:25,887 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T12:28:25,887 INFO [RS:0;1ba5af43fd99:35527 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-27T12:28:25,887 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T12:28:25,887 INFO [RS:0;1ba5af43fd99:35527 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-27T12:28:25,887 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer(3091): Received CLOSE for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:25,888 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer(956): aborting server 1ba5af43fd99,35527,1732710489490 2024-11-27T12:28:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41787 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:51760 deadline: 1732710565887, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ba5af43fd99 port=35527 startCode=1732710489490. As of locationSeqNum=12. 2024-11-27T12:28:25,888 INFO [RS:0;1ba5af43fd99:35527 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T12:28:25,888 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5556f2e573f63651ecac0666fe4456cb, disabling compactions & flushes 2024-11-27T12:28:25,888 INFO [RS:0;1ba5af43fd99:35527 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1ba5af43fd99:35527. 2024-11-27T12:28:25,888 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:25,888 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:25,888 DEBUG [RS:0;1ba5af43fd99:35527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:28:25,888 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. after waiting 0 ms 2024-11-27T12:28:25,888 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,41787,1732710489587, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,41787,1732710489587, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ba5af43fd99 port=35527 startCode=1732710489490. As of locationSeqNum=12. 2024-11-27T12:28:25,888 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:25,888 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,41787,1732710489587, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ba5af43fd99 port=35527 startCode=1732710489490. As of locationSeqNum=12. 2024-11-27T12:28:25,888 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,41787,1732710489587, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,35527,1732710489490, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ba5af43fd99 port=35527 startCode=1732710489490. As of locationSeqNum=12. 2024-11-27T12:28:25,889 DEBUG [RS:0;1ba5af43fd99:35527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:28:25,889 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T12:28:25,889 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T12:28:25,889 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T12:28:25,889 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T12:28:25,890 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-27T12:28:25,890 DEBUG [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 5556f2e573f63651ecac0666fe4456cb=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.} 2024-11-27T12:28:25,890 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T12:28:25,890 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T12:28:25,890 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T12:28:25,889 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ba5af43fd99,35527,1732710489490 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:25,890 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T12:28:25,890 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T12:28:25,890 DEBUG [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:25,891 ERROR [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-27T12:28:25,891 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T12:28:25,891 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:25,891 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T12:28:25,891 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5556f2e573f63651ecac0666fe4456cb: Waiting for close lock at 1732710505888Running coprocessor pre-close hooks at 1732710505888Disabling compacts and flushes for region at 1732710505888Disabling writes for close at 1732710505888Writing region close event to WAL at 1732710505891 (+3 ms)Running coprocessor post-close hooks at 1732710505891Closed at 1732710505891 2024-11-27T12:28:25,891 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ba5af43fd99,35527,1732710489490 aborting 2024-11-27T12:28:25,891 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732710505890Running coprocessor pre-close hooks at 1732710505890Disabling compacts and flushes for region at 1732710505890Disabling writes for close at 1732710505890Writing region close event to WAL at 1732710505891 (+1 ms)Running coprocessor post-close hooks at 1732710505891Closed at 1732710505891 2024-11-27T12:28:25,891 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ba5af43fd99,35527,1732710489490 aborting 2024-11-27T12:28:25,891 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:25,891 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1 from cache 2024-11-27T12:28:25,891 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T12:28:25,933 INFO [regionserver/1ba5af43fd99:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T12:28:25,993 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:28:25,995 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1] 2024-11-27T12:28:25,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ba5af43fd99,35527,1732710489490 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:25,998 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ba5af43fd99,35527,1732710489490 aborting 2024-11-27T12:28:25,998 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ba5af43fd99,35527,1732710489490 aborting 2024-11-27T12:28:25,998 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1 from cache 2024-11-27T12:28:26,019 INFO [regionserver/1ba5af43fd99:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-27T12:28:26,019 INFO [regionserver/1ba5af43fd99:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-27T12:28:26,091 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer(976): stopping server 1ba5af43fd99,35527,1732710489490; all regions closed. 2024-11-27T12:28:26,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741836_1012 (size=6845) 2024-11-27T12:28:26,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741836_1012 (size=6845) 2024-11-27T12:28:26,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741836_1012 (size=6845) 2024-11-27T12:28:26,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741834_1010 (size=2142) 2024-11-27T12:28:26,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741834_1010 (size=2142) 2024-11-27T12:28:26,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741834_1010 (size=2142) 2024-11-27T12:28:26,103 DEBUG [RS:0;1ba5af43fd99:35527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:28:26,103 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T12:28:26,103 INFO [RS:0;1ba5af43fd99:35527 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T12:28:26,103 INFO [RS:0;1ba5af43fd99:35527 {}] hbase.ChoreService(370): Chore service for: regionserver/1ba5af43fd99:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T12:28:26,103 INFO [RS:0;1ba5af43fd99:35527 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T12:28:26,104 INFO [regionserver/1ba5af43fd99:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T12:28:26,104 INFO [RS:0;1ba5af43fd99:35527 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35527 2024-11-27T12:28:26,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35527-0x1017c43d3070001, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1ba5af43fd99,35527,1732710489490 2024-11-27T12:28:26,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T12:28:26,180 INFO [RS:0;1ba5af43fd99:35527 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T12:28:26,183 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1ba5af43fd99,35527,1732710489490] 2024-11-27T12:28:26,200 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1ba5af43fd99,35527,1732710489490 already deleted, retry=false 2024-11-27T12:28:26,200 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 1ba5af43fd99,35527,1732710489490 on 1ba5af43fd99,46311,1732710488723 2024-11-27T12:28:26,204 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:28:26,206 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1] 2024-11-27T12:28:26,207 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 1ba5af43fd99,35527,1732710489490, splitWal=true, meta=true 2024-11-27T12:28:26,208 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 1ba5af43fd99:35527 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 1ba5af43fd99/172.17.0.3:35527 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:26,210 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1, error=java.net.ConnectException: Call to address=1ba5af43fd99:35527 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 1ba5af43fd99/172.17.0.3:35527 2024-11-27T12:28:26,210 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-27T12:28:26,210 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1 from cache 2024-11-27T12:28:26,211 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 1ba5af43fd99:35527 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 1ba5af43fd99/172.17.0.3:35527 2024-11-27T12:28:26,212 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 1ba5af43fd99,35527,1732710489490 (carryingMeta=true) 1ba5af43fd99,35527,1732710489490/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@53dd8abe[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-27T12:28:26,212 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 1ba5af43fd99,35527,1732710489490, splitWal=true, meta=true 2024-11-27T12:28:26,214 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 1ba5af43fd99,35527,1732710489490, splitWal=true, meta=true, isMeta: true 2024-11-27T12:28:26,215 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting 2024-11-27T12:28:26,216 INFO [PEWorker-2 {}] master.SplitWALManager(105): 1ba5af43fd99,35527,1732710489490 WAL count=1, meta=true 2024-11-27T12:28:26,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta}] 2024-11-27T12:28:26,224 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=1ba5af43fd99,41787,1732710489587 2024-11-27T12:28:26,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta, worker=1ba5af43fd99,41787,1732710489587}] 2024-11-27T12:28:26,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35527-0x1017c43d3070001, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:28:26,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35527-0x1017c43d3070001, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:28:26,291 INFO [RS:0;1ba5af43fd99:35527 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T12:28:26,291 INFO [RS:0;1ba5af43fd99:35527 {}] regionserver.HRegionServer(1031): Exiting; stopping=1ba5af43fd99,35527,1732710489490; zookeeper connection closed. 2024-11-27T12:28:26,291 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@33546e47 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@33546e47 2024-11-27T12:28:26,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41787 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-27T12:28:26,411 INFO [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta, size=6.7 K (6845bytes) 2024-11-27T12:28:26,411 INFO [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta 2024-11-27T12:28:26,412 INFO [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta after 1ms 2024-11-27T12:28:26,414 INFO [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta took 3ms 2024-11-27T12:28:26,420 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-27T12:28:26,421 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta so closing down 2024-11-27T12:28:26,421 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T12:28:26,422 INFO [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta.temp 2024-11-27T12:28:26,423 INFO [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta.temp 2024-11-27T12:28:26,424 INFO [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T12:28:26,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741892_1070 (size=6604) 2024-11-27T12:28:26,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741892_1070 (size=6604) 2024-11-27T12:28:26,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741892_1070 (size=6604) 2024-11-27T12:28:26,430 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-27T12:28:26,432 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta.temp to hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-27T12:28:26,432 INFO [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 18 ms; skipped=1; WAL=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false 2024-11-27T12:28:26,432 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta, journal: Splitting hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta, size=6.7 K (6845bytes) at 1732710506411Finishing writing output for hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta so closing down at 1732710506421 (+10 ms)Creating recovered edits writer path=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta.temp at 1732710506423 (+2 ms)3 split writer threads finished at 1732710506424 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732710506430 (+6 ms)Rename recovered edits hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta.temp to hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1732710506432 (+2 ms)Processed 16 edits across 1 Regions in 18 ms; skipped=1; WAL=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false at 1732710506432 2024-11-27T12:28:26,432 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta 2024-11-27T12:28:26,433 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-27T12:28:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46311 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-27T12:28:26,439 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta to hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/oldWALs 2024-11-27T12:28:26,442 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-27T12:28:26,442 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta, worker=1ba5af43fd99,41787,1732710489587 in 213 msec 2024-11-27T12:28:26,443 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=1ba5af43fd99,41787,1732710489587 2024-11-27T12:28:26,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-27T12:28:26,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 1ba5af43fd99%2C35527%2C1732710489490.meta.1732710491557.meta, worker=1ba5af43fd99,41787,1732710489587 in 226 msec 2024-11-27T12:28:26,448 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting dir is empty, no logs to split. 2024-11-27T12:28:26,448 INFO [PEWorker-2 {}] master.SplitWALManager(105): 1ba5af43fd99,35527,1732710489490 WAL count=0, meta=true 2024-11-27T12:28:26,448 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 1ba5af43fd99,35527,1732710489490 WAL splitting is done? wals=0, meta=true 2024-11-27T12:28:26,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T12:28:26,451 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T12:28:26,453 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T12:28:26,513 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:28:26,515 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1] 2024-11-27T12:28:26,516 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 1ba5af43fd99:35527 this server is in the failed servers list 2024-11-27T12:28:26,516 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=1ba5af43fd99:35527 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ba5af43fd99:35527 2024-11-27T12:28:26,516 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ba5af43fd99:35527 2024-11-27T12:28:26,516 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1 from cache 2024-11-27T12:28:26,603 DEBUG [1ba5af43fd99:46311 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-27T12:28:26,604 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(204): Hosts are {1ba5af43fd99=0} racks are {/default-rack=0} 2024-11-27T12:28:26,604 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T12:28:26,604 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T12:28:26,604 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T12:28:26,604 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T12:28:26,604 INFO [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T12:28:26,604 INFO [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T12:28:26,604 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T12:28:26,605 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1ba5af43fd99,44849,1732710489640 2024-11-27T12:28:26,607 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1ba5af43fd99,44849,1732710489640, state=OPENING 2024-11-27T12:28:26,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:28:26,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:28:26,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:28:26,622 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T12:28:26,622 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:28:26,622 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:28:26,622 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:28:26,622 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1ba5af43fd99,44849,1732710489640}] 2024-11-27T12:28:26,778 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T12:28:26,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34367, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T12:28:26,793 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T12:28:26,793 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:26,793 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T12:28:26,795 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ba5af43fd99%2C44849%2C1732710489640.meta, suffix=.meta, logDir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,44849,1732710489640, archiveDir=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/oldWALs, maxLogs=32 2024-11-27T12:28:26,808 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,44849,1732710489640/1ba5af43fd99%2C44849%2C1732710489640.meta.1732710506795.meta, exclude list is [], retry=0 2024-11-27T12:28:26,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:26,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:26,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:26,814 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,44849,1732710489640/1ba5af43fd99%2C44849%2C1732710489640.meta.1732710506795.meta 2024-11-27T12:28:26,814 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:46317:46317)] 2024-11-27T12:28:26,814 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:26,815 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T12:28:26,815 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T12:28:26,815 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T12:28:26,815 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T12:28:26,815 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:26,815 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T12:28:26,815 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T12:28:26,818 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T12:28:26,819 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T12:28:26,819 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:26,819 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:28:26,820 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T12:28:26,820 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T12:28:26,820 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:26,821 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:28:26,821 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T12:28:26,822 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T12:28:26,822 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:26,822 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:28:26,822 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T12:28:26,823 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T12:28:26,823 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:26,824 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:28:26,824 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T12:28:26,824 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740 2024-11-27T12:28:26,826 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740 2024-11-27T12:28:26,826 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-27T12:28:26,830 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-27T12:28:26,831 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-27T12:28:26,849 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/.tmp/info/7c7b063d6bf4465187d3c40a6fda6014 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb./info:regioninfo/1732710505798/Put/seqid=0 2024-11-27T12:28:26,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741894_1072 (size=11177) 2024-11-27T12:28:26,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741894_1072 (size=11177) 2024-11-27T12:28:26,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741894_1072 (size=11177) 2024-11-27T12:28:26,857 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/.tmp/info/7c7b063d6bf4465187d3c40a6fda6014 2024-11-27T12:28:26,881 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/.tmp/ns/d7d0059dc5a542a6ba33894507c84fd3 is 43, key is default/ns:d/1732710491712/Put/seqid=0 2024-11-27T12:28:26,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741895_1073 (size=5153) 2024-11-27T12:28:26,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741895_1073 (size=5153) 2024-11-27T12:28:26,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741895_1073 (size=5153) 2024-11-27T12:28:26,889 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/.tmp/ns/d7d0059dc5a542a6ba33894507c84fd3 2024-11-27T12:28:26,909 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/.tmp/table/6b7ee49f9c2f4a1ca7e4d9a837894759 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732710504088/Put/seqid=0 2024-11-27T12:28:26,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741896_1074 (size=5431) 2024-11-27T12:28:26,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741896_1074 (size=5431) 2024-11-27T12:28:26,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741896_1074 (size=5431) 2024-11-27T12:28:26,916 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/.tmp/table/6b7ee49f9c2f4a1ca7e4d9a837894759 2024-11-27T12:28:26,923 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/.tmp/info/7c7b063d6bf4465187d3c40a6fda6014 as hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/info/7c7b063d6bf4465187d3c40a6fda6014 2024-11-27T12:28:26,930 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/info/7c7b063d6bf4465187d3c40a6fda6014, entries=36, sequenceid=18, filesize=10.9 K 2024-11-27T12:28:26,931 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/.tmp/ns/d7d0059dc5a542a6ba33894507c84fd3 as hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/ns/d7d0059dc5a542a6ba33894507c84fd3 2024-11-27T12:28:26,938 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/ns/d7d0059dc5a542a6ba33894507c84fd3, entries=2, sequenceid=18, filesize=5.0 K 2024-11-27T12:28:26,939 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/.tmp/table/6b7ee49f9c2f4a1ca7e4d9a837894759 as hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/table/6b7ee49f9c2f4a1ca7e4d9a837894759 2024-11-27T12:28:26,947 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/table/6b7ee49f9c2f4a1ca7e4d9a837894759, entries=2, sequenceid=18, filesize=5.3 K 2024-11-27T12:28:26,947 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 116ms, sequenceid=18, compaction requested=false; wal=null 2024-11-27T12:28:26,948 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-27T12:28:26,949 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T12:28:26,949 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T12:28:26,950 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T12:28:26,951 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T12:28:26,954 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-27T12:28:26,955 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70136926, jitterRate=0.04512163996696472}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T12:28:26,955 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T12:28:26,955 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732710506816Writing region info on filesystem at 1732710506816Initializing all the Stores at 1732710506816Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732710506817 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732710506818 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710506818Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732710506818Obtaining lock to block concurrent updates at 1732710506831 (+13 ms)Preparing flush snapshotting stores in 1588230740 at 1732710506831Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1732710506831Flushing stores of hbase:meta,,1.1588230740 at 1732710506831Flushing 1588230740/info: creating writer at 1732710506831Flushing 1588230740/info: appending metadata at 1732710506849 (+18 ms)Flushing 1588230740/info: closing flushed file at 1732710506849Flushing 1588230740/ns: creating writer at 1732710506866 (+17 ms)Flushing 1588230740/ns: appending metadata at 1732710506881 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732710506881Flushing 1588230740/table: creating writer at 1732710506894 (+13 ms)Flushing 1588230740/table: appending metadata at 1732710506909 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732710506909Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39cfd6b4: reopening flushed file at 1732710506922 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b508838: reopening flushed file at 1732710506930 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@747a242a: reopening flushed file at 1732710506938 (+8 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 116ms, sequenceid=18, compaction requested=false; wal=null at 1732710506947 (+9 ms)Cleaning up temporary data from old regions at 1732710506949 (+2 ms)Running coprocessor post-open hooks at 1732710506955 (+6 ms)Region opened successfully at 1732710506955 2024-11-27T12:28:26,956 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1732710506778 2024-11-27T12:28:26,959 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T12:28:26,959 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T12:28:26,960 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=1ba5af43fd99,44849,1732710489640 2024-11-27T12:28:26,961 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1ba5af43fd99,44849,1732710489640, state=OPEN 2024-11-27T12:28:26,963 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46311 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1ba5af43fd99,44849,1732710489640, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T12:28:26,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:28:26,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:28:26,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:28:26,969 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:28:26,969 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:28:26,969 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:28:26,969 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1ba5af43fd99,44849,1732710489640 2024-11-27T12:28:26,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-27T12:28:26,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1ba5af43fd99,44849,1732710489640 in 347 msec 2024-11-27T12:28:26,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-27T12:28:26,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 522 msec 2024-11-27T12:28:26,975 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 1ba5af43fd99,35527,1732710489490 had 2 regions 2024-11-27T12:28:26,976 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 1ba5af43fd99,35527,1732710489490, splitWal=true, meta=true, isMeta: false 2024-11-27T12:28:26,978 INFO [PEWorker-1 {}] master.SplitWALManager(105): 1ba5af43fd99,35527,1732710489490 WAL count=1, meta=false 2024-11-27T12:28:26,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 1ba5af43fd99%2C35527%2C1732710489490.1732710491157}] 2024-11-27T12:28:26,980 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=1ba5af43fd99,41787,1732710489587 2024-11-27T12:28:26,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 1ba5af43fd99%2C35527%2C1732710489490.1732710491157, worker=1ba5af43fd99,41787,1732710489587}] 2024-11-27T12:28:27,024 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:28:27,025 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,44849,1732710489640, seqNum=-1] 2024-11-27T12:28:27,026 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:28:27,029 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:28:27,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41787 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-27T12:28:27,144 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 1ba5af43fd99:35527 this server is in the failed servers list 2024-11-27T12:28:27,144 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,35527,1732710489490, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,35527,1732710489490, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=1ba5af43fd99:35527 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ba5af43fd99:35527 2024-11-27T12:28:27,145 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,35527,1732710489490, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ba5af43fd99:35527 2024-11-27T12:28:27,145 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,35527,1732710489490, seqNum=12 from cache 2024-11-27T12:28:27,158 INFO [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.1732710491157, size=2.1 K (2142bytes) 2024-11-27T12:28:27,159 INFO [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.1732710491157 2024-11-27T12:28:27,159 INFO [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.1732710491157 after 0ms 2024-11-27T12:28:27,161 INFO [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.1732710491157 took 3ms 2024-11-27T12:28:27,164 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 5556f2e573f63651ecac0666fe4456cb: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-27T12:28:27,164 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.1732710491157 so closing down 2024-11-27T12:28:27,164 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T12:28:27,164 INFO [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T12:28:27,164 INFO [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.1732710491157, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-11-27T12:28:27,164 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.1732710491157, journal: Splitting hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.1732710491157, size=2.1 K (2142bytes) at 1732710507159Finishing writing output for hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.1732710491157 so closing down at 1732710507164 (+5 ms)3 split writer threads finished at 1732710507164Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.1732710491157, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1732710507164 2024-11-27T12:28:27,165 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.1732710491157 2024-11-27T12:28:27,165 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-27T12:28:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46311 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-27T12:28:27,169 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting/1ba5af43fd99%2C35527%2C1732710489490.1732710491157 to hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/oldWALs 2024-11-27T12:28:27,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-27T12:28:27,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1ba5af43fd99%2C35527%2C1732710489490.1732710491157, worker=1ba5af43fd99,41787,1732710489587 in 188 msec 2024-11-27T12:28:27,173 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=1ba5af43fd99,41787,1732710489587 2024-11-27T12:28:27,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-27T12:28:27,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 1ba5af43fd99%2C35527%2C1732710489490.1732710491157, worker=1ba5af43fd99,41787,1732710489587 in 195 msec 2024-11-27T12:28:27,178 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/WALs/1ba5af43fd99,35527,1732710489490-splitting dir is empty, no logs to split. 2024-11-27T12:28:27,178 INFO [PEWorker-1 {}] master.SplitWALManager(105): 1ba5af43fd99,35527,1732710489490 WAL count=0, meta=false 2024-11-27T12:28:27,178 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 1ba5af43fd99,35527,1732710489490 WAL splitting is done? wals=0, meta=false 2024-11-27T12:28:27,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, ASSIGN}] 2024-11-27T12:28:27,182 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, ASSIGN 2024-11-27T12:28:27,183 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T12:28:27,333 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(204): Hosts are {1ba5af43fd99=0} racks are {/default-rack=0} 2024-11-27T12:28:27,334 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T12:28:27,334 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T12:28:27,334 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T12:28:27,334 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T12:28:27,334 INFO [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T12:28:27,334 INFO [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T12:28:27,334 DEBUG [1ba5af43fd99:46311 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T12:28:27,334 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=5556f2e573f63651ecac0666fe4456cb, regionState=OPENING, regionLocation=1ba5af43fd99,41787,1732710489587 2024-11-27T12:28:27,336 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 1ba5af43fd99:35527 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1ba5af43fd99/172.17.0.3:35527 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:27,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1, error=java.net.ConnectException: Call to address=1ba5af43fd99:35527 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1ba5af43fd99/172.17.0.3:35527 2024-11-27T12:28:27,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-27T12:28:27,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,35527,1732710489490, seqNum=-1 from cache 2024-11-27T12:28:27,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 1ba5af43fd99:35527 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1ba5af43fd99/172.17.0.3:35527 2024-11-27T12:28:27,361 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,35527,1732710489490, seqNum=18] 2024-11-27T12:28:27,362 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] ipc.AbstractRpcClient(357): Not trying to connect to 1ba5af43fd99:35527 this server is in the failed servers list 2024-11-27T12:28:27,362 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,35527,1732710489490, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,35527,1732710489490, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=1ba5af43fd99:35527 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ba5af43fd99:35527 2024-11-27T12:28:27,362 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,35527,1732710489490, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ba5af43fd99:35527 2024-11-27T12:28:27,362 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,35527,1732710489490, seqNum=18 from cache 2024-11-27T12:28:27,444 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:28:27,444 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,44849,1732710489640, seqNum=-1] 2024-11-27T12:28:27,444 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:28:27,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42739, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:28:27,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, ASSIGN because future has completed 2024-11-27T12:28:27,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5556f2e573f63651ecac0666fe4456cb, server=1ba5af43fd99,41787,1732710489587}] 2024-11-27T12:28:27,607 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:27,608 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 5556f2e573f63651ecac0666fe4456cb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:27,609 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:27,609 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:27,609 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:27,609 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:27,612 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:27,613 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5556f2e573f63651ecac0666fe4456cb columnFamilyName cf1 2024-11-27T12:28:27,613 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:27,621 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf1/35bfd4236e9d440bb96762e929a69bc4 2024-11-27T12:28:27,621 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(327): Store=5556f2e573f63651ecac0666fe4456cb/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:27,621 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:27,622 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5556f2e573f63651ecac0666fe4456cb columnFamilyName cf2 2024-11-27T12:28:27,622 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:27,629 DEBUG [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/cf2/f8b52a6c9b3e4d6585fad73fbd1821cc 2024-11-27T12:28:27,629 INFO [StoreOpener-5556f2e573f63651ecac0666fe4456cb-1 {}] regionserver.HStore(327): Store=5556f2e573f63651ecac0666fe4456cb/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:27,629 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:27,630 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:27,631 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:27,632 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:27,632 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:27,632 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-27T12:28:27,633 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:27,634 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 5556f2e573f63651ecac0666fe4456cb; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73524804, jitterRate=0.09560495615005493}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-27T12:28:27,634 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:27,635 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 5556f2e573f63651ecac0666fe4456cb: Running coprocessor pre-open hook at 1732710507610Writing region info on filesystem at 1732710507610Initializing all the Stores at 1732710507611 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710507611Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710507612 (+1 ms)Cleaning up temporary data from old regions at 1732710507632 (+20 ms)Running coprocessor post-open hooks at 1732710507634 (+2 ms)Region opened successfully at 1732710507635 (+1 ms) 2024-11-27T12:28:27,635 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., pid=21, masterSystemTime=1732710507602 2024-11-27T12:28:27,637 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:27,638 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:27,638 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=5556f2e573f63651ecac0666fe4456cb, regionState=OPEN, openSeqNum=18, regionLocation=1ba5af43fd99,41787,1732710489587 2024-11-27T12:28:27,640 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5556f2e573f63651ecac0666fe4456cb, server=1ba5af43fd99,41787,1732710489587 because future has completed 2024-11-27T12:28:27,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-27T12:28:27,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 5556f2e573f63651ecac0666fe4456cb, server=1ba5af43fd99,41787,1732710489587 in 193 msec 2024-11-27T12:28:27,647 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-27T12:28:27,647 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server 1ba5af43fd99,35527,1732710489490 after splitting done 2024-11-27T12:28:27,647 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5556f2e573f63651ecac0666fe4456cb, ASSIGN in 464 msec 2024-11-27T12:28:27,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 1ba5af43fd99,35527,1732710489490, splitWal=true, meta=true in 1.4460 sec 2024-11-27T12:28:27,667 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb., hostname=1ba5af43fd99,41787,1732710489587, seqNum=18] 2024-11-27T12:28:27,682 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=414 (was 419), OpenFileDescriptor=1052 (was 1027) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=274 (was 298), ProcessCount=11 (was 11), AvailableMemoryMB=8790 (was 8884) 2024-11-27T12:28:27,682 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1052 is superior to 1024 2024-11-27T12:28:27,694 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=414, OpenFileDescriptor=1052, MaxFileDescriptor=1048576, SystemLoadAverage=274, ProcessCount=11, AvailableMemoryMB=8789 2024-11-27T12:28:27,694 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1052 is superior to 1024 2024-11-27T12:28:27,708 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:27,710 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:27,710 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:28:27,713 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-79881556, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/hregion-79881556, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:27,727 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-79881556/hregion-79881556.1732710507713, exclude list is [], retry=0 2024-11-27T12:28:27,730 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:27,730 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:27,730 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:27,732 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-79881556/hregion-79881556.1732710507713 2024-11-27T12:28:27,732 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:46317:46317)] 2024-11-27T12:28:27,732 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5da2cd8d301cadf90ebfecb7ecd8100f, NAME => 'testReplayEditsWrittenViaHRegion,,1732710507708.5da2cd8d301cadf90ebfecb7ecd8100f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38485/hbase 2024-11-27T12:28:27,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741898_1076 (size=67) 2024-11-27T12:28:27,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741898_1076 (size=67) 2024-11-27T12:28:27,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741898_1076 (size=67) 2024-11-27T12:28:27,741 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732710507708.5da2cd8d301cadf90ebfecb7ecd8100f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:27,743 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,744 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5da2cd8d301cadf90ebfecb7ecd8100f columnFamilyName a 2024-11-27T12:28:27,744 DEBUG [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:27,745 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.HStore(327): Store=5da2cd8d301cadf90ebfecb7ecd8100f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:27,745 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,746 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5da2cd8d301cadf90ebfecb7ecd8100f columnFamilyName b 2024-11-27T12:28:27,746 DEBUG [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:27,747 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.HStore(327): Store=5da2cd8d301cadf90ebfecb7ecd8100f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:27,747 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,748 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5da2cd8d301cadf90ebfecb7ecd8100f columnFamilyName c 2024-11-27T12:28:27,748 DEBUG [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:27,749 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.HStore(327): Store=5da2cd8d301cadf90ebfecb7ecd8100f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:27,749 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,750 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,750 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,752 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,752 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,752 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T12:28:27,753 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,755 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:28:27,756 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5da2cd8d301cadf90ebfecb7ecd8100f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67704954, jitterRate=0.00888243317604065}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T12:28:27,756 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5da2cd8d301cadf90ebfecb7ecd8100f: Writing region info on filesystem at 1732710507742Initializing all the Stores at 1732710507742Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710507742Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710507743 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710507743Cleaning up temporary data from old regions at 1732710507752 (+9 ms)Region opened successfully at 1732710507756 (+4 ms) 2024-11-27T12:28:27,756 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5da2cd8d301cadf90ebfecb7ecd8100f, disabling compactions & flushes 2024-11-27T12:28:27,756 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732710507708.5da2cd8d301cadf90ebfecb7ecd8100f. 2024-11-27T12:28:27,756 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732710507708.5da2cd8d301cadf90ebfecb7ecd8100f. 2024-11-27T12:28:27,756 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732710507708.5da2cd8d301cadf90ebfecb7ecd8100f. after waiting 0 ms 2024-11-27T12:28:27,757 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732710507708.5da2cd8d301cadf90ebfecb7ecd8100f. 2024-11-27T12:28:27,757 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732710507708.5da2cd8d301cadf90ebfecb7ecd8100f. 2024-11-27T12:28:27,757 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5da2cd8d301cadf90ebfecb7ecd8100f: Waiting for close lock at 1732710507756Disabling compacts and flushes for region at 1732710507756Disabling writes for close at 1732710507756Writing region close event to WAL at 1732710507757 (+1 ms)Closed at 1732710507757 2024-11-27T12:28:27,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741897_1075 (size=93) 2024-11-27T12:28:27,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741897_1075 (size=93) 2024-11-27T12:28:27,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741897_1075 (size=93) 2024-11-27T12:28:27,791 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T12:28:27,791 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-79881556:(num 1732710507713) 2024-11-27T12:28:27,791 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:27,793 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:27,806 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707/wal.1732710507793, exclude list is [], retry=0 2024-11-27T12:28:27,808 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:27,809 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:27,809 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:27,810 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707/wal.1732710507793 2024-11-27T12:28:27,810 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:34427:34427)] 2024-11-27T12:28:27,811 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5da2cd8d301cadf90ebfecb7ecd8100f, NAME => 'testReplayEditsWrittenViaHRegion,,1732710507708.5da2cd8d301cadf90ebfecb7ecd8100f.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:27,811 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732710507708.5da2cd8d301cadf90ebfecb7ecd8100f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:27,811 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,811 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,812 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,813 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5da2cd8d301cadf90ebfecb7ecd8100f columnFamilyName a 2024-11-27T12:28:27,813 DEBUG [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:27,813 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.HStore(327): Store=5da2cd8d301cadf90ebfecb7ecd8100f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:27,813 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,814 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5da2cd8d301cadf90ebfecb7ecd8100f columnFamilyName b 2024-11-27T12:28:27,814 DEBUG [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:27,814 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.HStore(327): Store=5da2cd8d301cadf90ebfecb7ecd8100f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:27,815 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,815 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5da2cd8d301cadf90ebfecb7ecd8100f columnFamilyName c 2024-11-27T12:28:27,815 DEBUG [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:27,815 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.HStore(327): Store=5da2cd8d301cadf90ebfecb7ecd8100f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:27,816 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,816 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,817 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,818 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,818 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,818 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T12:28:27,820 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:27,820 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5da2cd8d301cadf90ebfecb7ecd8100f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61538622, jitterRate=-0.08300307393074036}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T12:28:27,821 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5da2cd8d301cadf90ebfecb7ecd8100f: Writing region info on filesystem at 1732710507811Initializing all the Stores at 1732710507812 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710507812Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710507812Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710507812Cleaning up temporary data from old regions at 1732710507818 (+6 ms)Region opened successfully at 1732710507821 (+3 ms) 2024-11-27T12:28:27,842 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5da2cd8d301cadf90ebfecb7ecd8100f 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-27T12:28:27,858 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/.tmp/a/5646e6d46734476983d1e166855f0522 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732710507821/Put/seqid=0 2024-11-27T12:28:27,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741900_1078 (size=5958) 2024-11-27T12:28:27,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741900_1078 (size=5958) 2024-11-27T12:28:27,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741900_1078 (size=5958) 2024-11-27T12:28:27,864 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/.tmp/a/5646e6d46734476983d1e166855f0522 2024-11-27T12:28:27,887 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/.tmp/b/db4aba228b044a1ba1ed102c352df1df is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732710507828/Put/seqid=0 2024-11-27T12:28:27,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741901_1079 (size=5958) 2024-11-27T12:28:27,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741901_1079 (size=5958) 2024-11-27T12:28:27,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741901_1079 (size=5958) 2024-11-27T12:28:27,894 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/.tmp/b/db4aba228b044a1ba1ed102c352df1df 2024-11-27T12:28:27,913 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/.tmp/c/44624d55acf342edb532fc634ee6d521 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732710507834/Put/seqid=0 2024-11-27T12:28:27,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741902_1080 (size=5958) 2024-11-27T12:28:27,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741902_1080 (size=5958) 2024-11-27T12:28:27,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741902_1080 (size=5958) 2024-11-27T12:28:27,921 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/.tmp/c/44624d55acf342edb532fc634ee6d521 2024-11-27T12:28:27,927 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/.tmp/a/5646e6d46734476983d1e166855f0522 as hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/a/5646e6d46734476983d1e166855f0522 2024-11-27T12:28:27,933 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/a/5646e6d46734476983d1e166855f0522, entries=10, sequenceid=33, filesize=5.8 K 2024-11-27T12:28:27,934 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/.tmp/b/db4aba228b044a1ba1ed102c352df1df as hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/b/db4aba228b044a1ba1ed102c352df1df 2024-11-27T12:28:27,939 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/b/db4aba228b044a1ba1ed102c352df1df, entries=10, sequenceid=33, filesize=5.8 K 2024-11-27T12:28:27,940 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/.tmp/c/44624d55acf342edb532fc634ee6d521 as hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/c/44624d55acf342edb532fc634ee6d521 2024-11-27T12:28:27,945 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/c/44624d55acf342edb532fc634ee6d521, entries=10, sequenceid=33, filesize=5.8 K 2024-11-27T12:28:27,946 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 5da2cd8d301cadf90ebfecb7ecd8100f in 104ms, sequenceid=33, compaction requested=false 2024-11-27T12:28:27,947 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5da2cd8d301cadf90ebfecb7ecd8100f: 2024-11-27T12:28:27,947 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5da2cd8d301cadf90ebfecb7ecd8100f, disabling compactions & flushes 2024-11-27T12:28:27,947 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732710507708.5da2cd8d301cadf90ebfecb7ecd8100f. 2024-11-27T12:28:27,947 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732710507708.5da2cd8d301cadf90ebfecb7ecd8100f. 2024-11-27T12:28:27,947 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732710507708.5da2cd8d301cadf90ebfecb7ecd8100f. after waiting 0 ms 2024-11-27T12:28:27,947 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732710507708.5da2cd8d301cadf90ebfecb7ecd8100f. 2024-11-27T12:28:27,948 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732710507708.5da2cd8d301cadf90ebfecb7ecd8100f. 2024-11-27T12:28:27,948 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5da2cd8d301cadf90ebfecb7ecd8100f: Waiting for close lock at 1732710507947Disabling compacts and flushes for region at 1732710507947Disabling writes for close at 1732710507947Writing region close event to WAL at 1732710507948 (+1 ms)Closed at 1732710507948 2024-11-27T12:28:27,953 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707/wal.1732710507793 not finished, retry = 0 2024-11-27T12:28:27,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741899_1077 (size=5937) 2024-11-27T12:28:27,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741899_1077 (size=5937) 2024-11-27T12:28:27,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741899_1077 (size=5937) 2024-11-27T12:28:28,059 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/b/db4aba228b044a1ba1ed102c352df1df to hdfs://localhost:38485/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/b/db4aba228b044a1ba1ed102c352df1df 2024-11-27T12:28:28,083 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38485/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707/wal.1732710507793, size=5.8 K (5937bytes) 2024-11-27T12:28:28,083 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38485/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707/wal.1732710507793 2024-11-27T12:28:28,084 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38485/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707/wal.1732710507793 after 1ms 2024-11-27T12:28:28,086 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38485/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707/wal.1732710507793 took 3ms 2024-11-27T12:28:28,089 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38485/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707/wal.1732710507793 so closing down 2024-11-27T12:28:28,089 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T12:28:28,090 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732710507793.temp 2024-11-27T12:28:28,091 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/recovered.edits/0000000000000000003-wal.1732710507793.temp 2024-11-27T12:28:28,092 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T12:28:28,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741903_1081 (size=5223) 2024-11-27T12:28:28,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741903_1081 (size=5223) 2024-11-27T12:28:28,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741903_1081 (size=5223) 2024-11-27T12:28:28,100 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/recovered.edits/0000000000000000003-wal.1732710507793.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-27T12:28:28,102 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/recovered.edits/0000000000000000003-wal.1732710507793.temp to hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/recovered.edits/0000000000000000032 2024-11-27T12:28:28,102 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:38485/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707/wal.1732710507793, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-11-27T12:28:28,102 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38485/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707/wal.1732710507793, journal: Splitting hdfs://localhost:38485/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707/wal.1732710507793, size=5.8 K (5937bytes) at 1732710508083Finishing writing output for hdfs://localhost:38485/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707/wal.1732710507793 so closing down at 1732710508089 (+6 ms)Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/recovered.edits/0000000000000000003-wal.1732710507793.temp at 1732710508091 (+2 ms)3 split writer threads finished at 1732710508092 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/recovered.edits/0000000000000000003-wal.1732710507793.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732710508100 (+8 ms)Rename recovered edits hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/recovered.edits/0000000000000000003-wal.1732710507793.temp to hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/recovered.edits/0000000000000000032 at 1732710508102 (+2 ms)Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:38485/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707/wal.1732710507793, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1732710508102 2024-11-27T12:28:28,104 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38485/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707/wal.1732710507793 to hdfs://localhost:38485/hbase/oldWALs/wal.1732710507793 2024-11-27T12:28:28,105 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/recovered.edits/0000000000000000032 2024-11-27T12:28:28,105 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:28,107 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:28,120 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707/wal.1732710508107, exclude list is [], retry=0 2024-11-27T12:28:28,124 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:28,124 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:28,125 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:28,127 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707/wal.1732710508107 2024-11-27T12:28:28,128 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:45591:45591)] 2024-11-27T12:28:28,128 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5da2cd8d301cadf90ebfecb7ecd8100f, NAME => 'testReplayEditsWrittenViaHRegion,,1732710507708.5da2cd8d301cadf90ebfecb7ecd8100f.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:28,128 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732710507708.5da2cd8d301cadf90ebfecb7ecd8100f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:28,129 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:28,129 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:28,130 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:28,132 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5da2cd8d301cadf90ebfecb7ecd8100f columnFamilyName a 2024-11-27T12:28:28,132 DEBUG [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:28,138 DEBUG [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/a/5646e6d46734476983d1e166855f0522 2024-11-27T12:28:28,139 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.HStore(327): Store=5da2cd8d301cadf90ebfecb7ecd8100f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:28,139 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:28,141 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5da2cd8d301cadf90ebfecb7ecd8100f columnFamilyName b 2024-11-27T12:28:28,141 DEBUG [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:28,141 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.HStore(327): Store=5da2cd8d301cadf90ebfecb7ecd8100f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:28,142 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:28,143 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5da2cd8d301cadf90ebfecb7ecd8100f columnFamilyName c 2024-11-27T12:28:28,143 DEBUG [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:28,150 DEBUG [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/c/44624d55acf342edb532fc634ee6d521 2024-11-27T12:28:28,150 INFO [StoreOpener-5da2cd8d301cadf90ebfecb7ecd8100f-1 {}] regionserver.HStore(327): Store=5da2cd8d301cadf90ebfecb7ecd8100f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:28,151 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:28,152 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:28,154 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:28,154 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/recovered.edits/0000000000000000032 2024-11-27T12:28:28,159 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/recovered.edits/0000000000000000032 2024-11-27T12:28:28,159 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5da2cd8d301cadf90ebfecb7ecd8100f 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-27T12:28:28,173 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/.tmp/b/d7679861f3cd4f1b9fd00024076d96f7 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732710507828/Put/seqid=0 2024-11-27T12:28:28,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741905_1083 (size=5958) 2024-11-27T12:28:28,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741905_1083 (size=5958) 2024-11-27T12:28:28,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741905_1083 (size=5958) 2024-11-27T12:28:28,181 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/.tmp/b/d7679861f3cd4f1b9fd00024076d96f7 2024-11-27T12:28:28,188 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/.tmp/b/d7679861f3cd4f1b9fd00024076d96f7 as hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/b/d7679861f3cd4f1b9fd00024076d96f7 2024-11-27T12:28:28,194 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/b/d7679861f3cd4f1b9fd00024076d96f7, entries=10, sequenceid=32, filesize=5.8 K 2024-11-27T12:28:28,195 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 5da2cd8d301cadf90ebfecb7ecd8100f in 36ms, sequenceid=32, compaction requested=false; wal=null 2024-11-27T12:28:28,196 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/recovered.edits/0000000000000000032 2024-11-27T12:28:28,197 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:28,197 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:28,198 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T12:28:28,199 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5da2cd8d301cadf90ebfecb7ecd8100f 2024-11-27T12:28:28,202 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/5da2cd8d301cadf90ebfecb7ecd8100f/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-27T12:28:28,203 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5da2cd8d301cadf90ebfecb7ecd8100f; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60160613, jitterRate=-0.10353700816631317}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T12:28:28,203 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5da2cd8d301cadf90ebfecb7ecd8100f: Writing region info on filesystem at 1732710508129Initializing all the Stores at 1732710508130 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710508130Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710508130Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710508130Obtaining lock to block concurrent updates at 1732710508159 (+29 ms)Preparing flush snapshotting stores in 5da2cd8d301cadf90ebfecb7ecd8100f at 1732710508159Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732710507708.5da2cd8d301cadf90ebfecb7ecd8100f., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732710508159Flushing stores of testReplayEditsWrittenViaHRegion,,1732710507708.5da2cd8d301cadf90ebfecb7ecd8100f. at 1732710508159Flushing 5da2cd8d301cadf90ebfecb7ecd8100f/b: creating writer at 1732710508159Flushing 5da2cd8d301cadf90ebfecb7ecd8100f/b: appending metadata at 1732710508173 (+14 ms)Flushing 5da2cd8d301cadf90ebfecb7ecd8100f/b: closing flushed file at 1732710508173Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@274967f1: reopening flushed file at 1732710508187 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 5da2cd8d301cadf90ebfecb7ecd8100f in 36ms, sequenceid=32, compaction requested=false; wal=null at 1732710508195 (+8 ms)Cleaning up temporary data from old regions at 1732710508197 (+2 ms)Region opened successfully at 1732710508203 (+6 ms) 2024-11-27T12:28:28,227 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=421 (was 414) Potentially hanging thread: PacketResponder: BP-39017082-172.17.0.3-1732710484128:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:47840 [Waiting for operation #46] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:41916 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-39017082-172.17.0.3-1732710484128:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:42092 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:56644 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:48042 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:56430 [Waiting for operation #51] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-39017082-172.17.0.3-1732710484128:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1126 (was 1052) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=274 (was 274), ProcessCount=11 (was 11), AvailableMemoryMB=8777 (was 8789) 2024-11-27T12:28:28,227 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1126 is superior to 1024 2024-11-27T12:28:28,240 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=421, OpenFileDescriptor=1126, MaxFileDescriptor=1048576, SystemLoadAverage=274, ProcessCount=11, AvailableMemoryMB=8777 2024-11-27T12:28:28,240 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1126 is superior to 1024 2024-11-27T12:28:28,255 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:28,258 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:28,259 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:28:28,261 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-99470237, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/hregion-99470237, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:28,274 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-99470237/hregion-99470237.1732710508262, exclude list is [], retry=0 2024-11-27T12:28:28,277 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:28,277 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:28,277 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:28,280 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-99470237/hregion-99470237.1732710508262 2024-11-27T12:28:28,280 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:46317:46317)] 2024-11-27T12:28:28,280 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 776fb2c4776eac3562191fb2954c4262, NAME => 'testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38485/hbase 2024-11-27T12:28:28,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741907_1085 (size=68) 2024-11-27T12:28:28,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741907_1085 (size=68) 2024-11-27T12:28:28,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741907_1085 (size=68) 2024-11-27T12:28:28,295 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:28,296 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,298 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 776fb2c4776eac3562191fb2954c4262 columnFamilyName a 2024-11-27T12:28:28,298 DEBUG [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:28,299 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] regionserver.HStore(327): Store=776fb2c4776eac3562191fb2954c4262/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:28,299 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,301 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 776fb2c4776eac3562191fb2954c4262 columnFamilyName b 2024-11-27T12:28:28,301 DEBUG [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:28,302 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] regionserver.HStore(327): Store=776fb2c4776eac3562191fb2954c4262/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:28,302 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,303 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 776fb2c4776eac3562191fb2954c4262 columnFamilyName c 2024-11-27T12:28:28,304 DEBUG [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:28,304 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] regionserver.HStore(327): Store=776fb2c4776eac3562191fb2954c4262/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:28,304 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,305 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,305 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,306 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,306 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,307 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T12:28:28,308 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,310 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:28:28,310 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 776fb2c4776eac3562191fb2954c4262; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66145492, jitterRate=-0.014355361461639404}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T12:28:28,311 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 776fb2c4776eac3562191fb2954c4262: Writing region info on filesystem at 1732710508295Initializing all the Stores at 1732710508296 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710508296Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710508296Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710508296Cleaning up temporary data from old regions at 1732710508306 (+10 ms)Region opened successfully at 1732710508311 (+5 ms) 2024-11-27T12:28:28,311 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 776fb2c4776eac3562191fb2954c4262, disabling compactions & flushes 2024-11-27T12:28:28,311 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262. 2024-11-27T12:28:28,311 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262. 2024-11-27T12:28:28,311 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262. after waiting 0 ms 2024-11-27T12:28:28,311 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262. 2024-11-27T12:28:28,311 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262. 2024-11-27T12:28:28,311 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 776fb2c4776eac3562191fb2954c4262: Waiting for close lock at 1732710508311Disabling compacts and flushes for region at 1732710508311Disabling writes for close at 1732710508311Writing region close event to WAL at 1732710508311Closed at 1732710508311 2024-11-27T12:28:28,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741906_1084 (size=93) 2024-11-27T12:28:28,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741906_1084 (size=93) 2024-11-27T12:28:28,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741906_1084 (size=93) 2024-11-27T12:28:28,316 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T12:28:28,316 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-99470237:(num 1732710508262) 2024-11-27T12:28:28,316 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:28,318 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:28,329 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255/wal.1732710508318, exclude list is [], retry=0 2024-11-27T12:28:28,332 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:28,332 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:28,333 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:28,334 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255/wal.1732710508318 2024-11-27T12:28:28,334 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:45591:45591)] 2024-11-27T12:28:28,400 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 776fb2c4776eac3562191fb2954c4262, NAME => 'testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:28,402 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,402 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:28,402 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,402 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,404 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,405 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 776fb2c4776eac3562191fb2954c4262 columnFamilyName a 2024-11-27T12:28:28,405 DEBUG [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:28,406 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] regionserver.HStore(327): Store=776fb2c4776eac3562191fb2954c4262/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:28,406 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,407 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 776fb2c4776eac3562191fb2954c4262 columnFamilyName b 2024-11-27T12:28:28,407 DEBUG [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:28,407 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] regionserver.HStore(327): Store=776fb2c4776eac3562191fb2954c4262/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:28,407 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,408 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 776fb2c4776eac3562191fb2954c4262 columnFamilyName c 2024-11-27T12:28:28,408 DEBUG [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:28,409 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] regionserver.HStore(327): Store=776fb2c4776eac3562191fb2954c4262/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:28,409 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,410 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,411 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,412 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,412 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,412 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T12:28:28,414 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,415 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 776fb2c4776eac3562191fb2954c4262; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60478720, jitterRate=-0.09879684448242188}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T12:28:28,415 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:28,415 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 776fb2c4776eac3562191fb2954c4262: Running coprocessor pre-open hook at 1732710508402Writing region info on filesystem at 1732710508402Initializing all the Stores at 1732710508403 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710508403Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710508404 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710508404Cleaning up temporary data from old regions at 1732710508412 (+8 ms)Running coprocessor post-open hooks at 1732710508415 (+3 ms)Region opened successfully at 1732710508415 2024-11-27T12:28:28,428 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 776fb2c4776eac3562191fb2954c4262 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-27T12:28:28,429 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 776fb2c4776eac3562191fb2954c4262/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:29,430 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 776fb2c4776eac3562191fb2954c4262/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:29,568 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T12:28:29,600 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-27T12:28:29,601 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-27T12:28:29,601 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-27T12:28:29,601 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-27T12:28:29,602 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-27T12:28:29,602 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-27T12:28:29,602 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-27T12:28:29,602 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-27T12:28:30,430 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 776fb2c4776eac3562191fb2954c4262/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:31,432 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 776fb2c4776eac3562191fb2954c4262/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:32,434 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 776fb2c4776eac3562191fb2954c4262/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:33,434 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 776fb2c4776eac3562191fb2954c4262/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:34,436 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 776fb2c4776eac3562191fb2954c4262/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:35,437 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 776fb2c4776eac3562191fb2954c4262/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:36,438 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 776fb2c4776eac3562191fb2954c4262/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:37,439 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 776fb2c4776eac3562191fb2954c4262/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:37,442 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 776fb2c4776eac3562191fb2954c4262: 2024-11-27T12:28:37,442 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262. 2024-11-27T12:28:37,458 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 776fb2c4776eac3562191fb2954c4262: 2024-11-27T12:28:37,458 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-27T12:28:37,458 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 776fb2c4776eac3562191fb2954c4262, disabling compactions & flushes 2024-11-27T12:28:37,458 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262. 2024-11-27T12:28:37,458 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262. 2024-11-27T12:28:37,458 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262. after waiting 0 ms 2024-11-27T12:28:37,458 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262. 2024-11-27T12:28:37,458 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262. 2024-11-27T12:28:37,458 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262. 2024-11-27T12:28:37,459 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 776fb2c4776eac3562191fb2954c4262: Waiting for close lock at 1732710517458Running coprocessor pre-close hooks at 1732710517458Disabling compacts and flushes for region at 1732710517458Disabling writes for close at 1732710517458Writing region close event to WAL at 1732710517458Running coprocessor post-close hooks at 1732710517458Closed at 1732710517458 2024-11-27T12:28:37,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741908_1086 (size=3941) 2024-11-27T12:28:37,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741908_1086 (size=3941) 2024-11-27T12:28:37,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741908_1086 (size=3941) 2024-11-27T12:28:37,476 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38485/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255/wal.1732710508318, size=3.8 K (3941bytes) 2024-11-27T12:28:37,476 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38485/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255/wal.1732710508318 2024-11-27T12:28:37,477 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38485/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255/wal.1732710508318 after 1ms 2024-11-27T12:28:37,479 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38485/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255/wal.1732710508318 took 3ms 2024-11-27T12:28:37,481 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38485/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255/wal.1732710508318 so closing down 2024-11-27T12:28:37,481 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T12:28:37,482 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732710508318.temp 2024-11-27T12:28:37,483 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/recovered.edits/0000000000000000004-wal.1732710508318.temp 2024-11-27T12:28:37,484 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T12:28:37,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741909_1087 (size=2983) 2024-11-27T12:28:37,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741909_1087 (size=2983) 2024-11-27T12:28:37,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741909_1087 (size=2983) 2024-11-27T12:28:37,490 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/recovered.edits/0000000000000000004-wal.1732710508318.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-27T12:28:37,491 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/recovered.edits/0000000000000000004-wal.1732710508318.temp to hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/recovered.edits/0000000000000000026 2024-11-27T12:28:37,492 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 11 ms; skipped=3; WAL=hdfs://localhost:38485/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255/wal.1732710508318, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-11-27T12:28:37,492 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38485/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255/wal.1732710508318, journal: Splitting hdfs://localhost:38485/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255/wal.1732710508318, size=3.8 K (3941bytes) at 1732710517476Finishing writing output for hdfs://localhost:38485/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255/wal.1732710508318 so closing down at 1732710517481 (+5 ms)Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/recovered.edits/0000000000000000004-wal.1732710508318.temp at 1732710517484 (+3 ms)3 split writer threads finished at 1732710517484Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/recovered.edits/0000000000000000004-wal.1732710508318.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732710517490 (+6 ms)Rename recovered edits hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/recovered.edits/0000000000000000004-wal.1732710508318.temp to hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/recovered.edits/0000000000000000026 at 1732710517491 (+1 ms)Processed 23 edits across 1 Regions in 11 ms; skipped=3; WAL=hdfs://localhost:38485/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255/wal.1732710508318, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1732710517492 (+1 ms) 2024-11-27T12:28:37,493 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38485/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255/wal.1732710508318 to hdfs://localhost:38485/hbase/oldWALs/wal.1732710508318 2024-11-27T12:28:37,494 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/recovered.edits/0000000000000000026 2024-11-27T12:28:37,494 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:37,496 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:37,507 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255/wal.1732710517496, exclude list is [], retry=0 2024-11-27T12:28:37,510 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:37,510 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:37,511 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:37,512 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255/wal.1732710517496 2024-11-27T12:28:37,512 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:34427:34427)] 2024-11-27T12:28:37,513 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 776fb2c4776eac3562191fb2954c4262, NAME => 'testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:37,513 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:37,513 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:37,514 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:37,514 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:37,516 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:37,516 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 776fb2c4776eac3562191fb2954c4262 columnFamilyName a 2024-11-27T12:28:37,516 DEBUG [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:37,517 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] regionserver.HStore(327): Store=776fb2c4776eac3562191fb2954c4262/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:37,517 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:37,518 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 776fb2c4776eac3562191fb2954c4262 columnFamilyName b 2024-11-27T12:28:37,518 DEBUG [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:37,519 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] regionserver.HStore(327): Store=776fb2c4776eac3562191fb2954c4262/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:37,519 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:37,519 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 776fb2c4776eac3562191fb2954c4262 columnFamilyName c 2024-11-27T12:28:37,519 DEBUG [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:37,520 INFO [StoreOpener-776fb2c4776eac3562191fb2954c4262-1 {}] regionserver.HStore(327): Store=776fb2c4776eac3562191fb2954c4262/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:37,520 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:37,521 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:37,522 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:37,523 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/recovered.edits/0000000000000000026 2024-11-27T12:28:37,526 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/recovered.edits/0000000000000000026 2024-11-27T12:28:37,526 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 776fb2c4776eac3562191fb2954c4262 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-27T12:28:37,542 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/.tmp/a/60a606d4b8904e8fb635eec02f71336e is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732710517448/Put/seqid=0 2024-11-27T12:28:37,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741911_1089 (size=5523) 2024-11-27T12:28:37,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741911_1089 (size=5523) 2024-11-27T12:28:37,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741911_1089 (size=5523) 2024-11-27T12:28:37,549 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/.tmp/a/60a606d4b8904e8fb635eec02f71336e 2024-11-27T12:28:37,570 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/.tmp/b/286c61c1e7284f798681c94fb1c31bd6 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732710517443/Put/seqid=0 2024-11-27T12:28:37,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741912_1090 (size=5524) 2024-11-27T12:28:37,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741912_1090 (size=5524) 2024-11-27T12:28:37,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741912_1090 (size=5524) 2024-11-27T12:28:37,576 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/.tmp/b/286c61c1e7284f798681c94fb1c31bd6 2024-11-27T12:28:37,593 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/.tmp/c/cc5b12161b154b50ad932fc8d266fb77 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732710517446/Put/seqid=0 2024-11-27T12:28:37,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741913_1091 (size=5457) 2024-11-27T12:28:37,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741913_1091 (size=5457) 2024-11-27T12:28:37,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741913_1091 (size=5457) 2024-11-27T12:28:37,600 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/.tmp/c/cc5b12161b154b50ad932fc8d266fb77 2024-11-27T12:28:37,606 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/.tmp/a/60a606d4b8904e8fb635eec02f71336e as hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/a/60a606d4b8904e8fb635eec02f71336e 2024-11-27T12:28:37,611 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/a/60a606d4b8904e8fb635eec02f71336e, entries=7, sequenceid=26, filesize=5.4 K 2024-11-27T12:28:37,612 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/.tmp/b/286c61c1e7284f798681c94fb1c31bd6 as hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/b/286c61c1e7284f798681c94fb1c31bd6 2024-11-27T12:28:37,618 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/b/286c61c1e7284f798681c94fb1c31bd6, entries=7, sequenceid=26, filesize=5.4 K 2024-11-27T12:28:37,619 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/.tmp/c/cc5b12161b154b50ad932fc8d266fb77 as hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/c/cc5b12161b154b50ad932fc8d266fb77 2024-11-27T12:28:37,625 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/c/cc5b12161b154b50ad932fc8d266fb77, entries=6, sequenceid=26, filesize=5.3 K 2024-11-27T12:28:37,625 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 776fb2c4776eac3562191fb2954c4262 in 99ms, sequenceid=26, compaction requested=false; wal=null 2024-11-27T12:28:37,626 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/recovered.edits/0000000000000000026 2024-11-27T12:28:37,627 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:37,627 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:37,628 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T12:28:37,629 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:37,631 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testReplayEditsAfterAbortingFlush/776fb2c4776eac3562191fb2954c4262/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-27T12:28:37,633 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 776fb2c4776eac3562191fb2954c4262; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74915053, jitterRate=0.11632128059864044}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T12:28:37,633 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 776fb2c4776eac3562191fb2954c4262 2024-11-27T12:28:37,633 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 776fb2c4776eac3562191fb2954c4262: Running coprocessor pre-open hook at 1732710517514Writing region info on filesystem at 1732710517514Initializing all the Stores at 1732710517515 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710517515Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710517515Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710517515Obtaining lock to block concurrent updates at 1732710517526 (+11 ms)Preparing flush snapshotting stores in 776fb2c4776eac3562191fb2954c4262 at 1732710517526Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732710517526Flushing stores of testReplayEditsAfterAbortingFlush,,1732710508256.776fb2c4776eac3562191fb2954c4262. at 1732710517526Flushing 776fb2c4776eac3562191fb2954c4262/a: creating writer at 1732710517526Flushing 776fb2c4776eac3562191fb2954c4262/a: appending metadata at 1732710517542 (+16 ms)Flushing 776fb2c4776eac3562191fb2954c4262/a: closing flushed file at 1732710517542Flushing 776fb2c4776eac3562191fb2954c4262/b: creating writer at 1732710517555 (+13 ms)Flushing 776fb2c4776eac3562191fb2954c4262/b: appending metadata at 1732710517569 (+14 ms)Flushing 776fb2c4776eac3562191fb2954c4262/b: closing flushed file at 1732710517569Flushing 776fb2c4776eac3562191fb2954c4262/c: creating writer at 1732710517580 (+11 ms)Flushing 776fb2c4776eac3562191fb2954c4262/c: appending metadata at 1732710517593 (+13 ms)Flushing 776fb2c4776eac3562191fb2954c4262/c: closing flushed file at 1732710517593Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d8f1f8f: reopening flushed file at 1732710517605 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@cfc3f17: reopening flushed file at 1732710517612 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d2189ee: reopening flushed file at 1732710517618 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 776fb2c4776eac3562191fb2954c4262 in 99ms, sequenceid=26, compaction requested=false; wal=null at 1732710517625 (+7 ms)Cleaning up temporary data from old regions at 1732710517627 (+2 ms)Running coprocessor post-open hooks at 1732710517633 (+6 ms)Region opened successfully at 1732710517633 2024-11-27T12:28:37,654 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=421 (was 421), OpenFileDescriptor=1182 (was 1126) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=239 (was 274), ProcessCount=11 (was 11), AvailableMemoryMB=8771 (was 8777) 2024-11-27T12:28:37,654 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-27T12:28:37,666 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=421, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=239, ProcessCount=11, AvailableMemoryMB=8770 2024-11-27T12:28:37,666 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-27T12:28:37,680 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:37,681 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:37,682 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:28:37,684 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-19101165, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/hregion-19101165, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:37,696 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-19101165/hregion-19101165.1732710517684, exclude list is [], retry=0 2024-11-27T12:28:37,698 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:37,698 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:37,698 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:37,701 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-19101165/hregion-19101165.1732710517684 2024-11-27T12:28:37,701 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:45591:45591)] 2024-11-27T12:28:37,701 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5bccd99aa8cdbafe7bc7d0490c78c0b1, NAME => 'testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38485/hbase 2024-11-27T12:28:37,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741915_1093 (size=61) 2024-11-27T12:28:37,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741915_1093 (size=61) 2024-11-27T12:28:37,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741915_1093 (size=61) 2024-11-27T12:28:37,713 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:37,714 INFO [StoreOpener-5bccd99aa8cdbafe7bc7d0490c78c0b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:37,716 INFO [StoreOpener-5bccd99aa8cdbafe7bc7d0490c78c0b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bccd99aa8cdbafe7bc7d0490c78c0b1 columnFamilyName a 2024-11-27T12:28:37,716 DEBUG [StoreOpener-5bccd99aa8cdbafe7bc7d0490c78c0b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:37,717 INFO [StoreOpener-5bccd99aa8cdbafe7bc7d0490c78c0b1-1 {}] regionserver.HStore(327): Store=5bccd99aa8cdbafe7bc7d0490c78c0b1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:37,717 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:37,717 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:37,718 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:37,718 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:37,718 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:37,720 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:37,722 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:28:37,722 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5bccd99aa8cdbafe7bc7d0490c78c0b1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73205322, jitterRate=0.0908443033695221}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T12:28:37,723 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5bccd99aa8cdbafe7bc7d0490c78c0b1: Writing region info on filesystem at 1732710517713Initializing all the Stores at 1732710517714 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710517714Cleaning up temporary data from old regions at 1732710517718 (+4 ms)Region opened successfully at 1732710517723 (+5 ms) 2024-11-27T12:28:37,723 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5bccd99aa8cdbafe7bc7d0490c78c0b1, disabling compactions & flushes 2024-11-27T12:28:37,723 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1. 2024-11-27T12:28:37,723 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1. 2024-11-27T12:28:37,723 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1. after waiting 0 ms 2024-11-27T12:28:37,723 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1. 2024-11-27T12:28:37,723 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1. 2024-11-27T12:28:37,723 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5bccd99aa8cdbafe7bc7d0490c78c0b1: Waiting for close lock at 1732710517723Disabling compacts and flushes for region at 1732710517723Disabling writes for close at 1732710517723Writing region close event to WAL at 1732710517723Closed at 1732710517723 2024-11-27T12:28:37,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741914_1092 (size=93) 2024-11-27T12:28:37,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741914_1092 (size=93) 2024-11-27T12:28:37,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741914_1092 (size=93) 2024-11-27T12:28:37,728 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T12:28:37,728 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-19101165:(num 1732710517684) 2024-11-27T12:28:37,728 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:37,730 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:37,741 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679/wal.1732710517730, exclude list is [], retry=0 2024-11-27T12:28:37,744 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:37,744 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:37,745 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:37,746 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679/wal.1732710517730 2024-11-27T12:28:37,746 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:45591:45591)] 2024-11-27T12:28:37,746 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5bccd99aa8cdbafe7bc7d0490c78c0b1, NAME => 'testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:37,746 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:37,746 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:37,746 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:37,748 INFO [StoreOpener-5bccd99aa8cdbafe7bc7d0490c78c0b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:37,748 INFO [StoreOpener-5bccd99aa8cdbafe7bc7d0490c78c0b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bccd99aa8cdbafe7bc7d0490c78c0b1 columnFamilyName a 2024-11-27T12:28:37,748 DEBUG [StoreOpener-5bccd99aa8cdbafe7bc7d0490c78c0b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:37,749 INFO [StoreOpener-5bccd99aa8cdbafe7bc7d0490c78c0b1-1 {}] regionserver.HStore(327): Store=5bccd99aa8cdbafe7bc7d0490c78c0b1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:37,749 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:37,749 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:37,751 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:37,751 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:37,751 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:37,752 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:37,753 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5bccd99aa8cdbafe7bc7d0490c78c0b1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71794909, jitterRate=0.06982751190662384}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T12:28:37,754 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5bccd99aa8cdbafe7bc7d0490c78c0b1: Writing region info on filesystem at 1732710517746Initializing all the Stores at 1732710517747 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710517747Cleaning up temporary data from old regions at 1732710517751 (+4 ms)Region opened successfully at 1732710517754 (+3 ms) 2024-11-27T12:28:37,763 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5bccd99aa8cdbafe7bc7d0490c78c0b1, disabling compactions & flushes 2024-11-27T12:28:37,763 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1. 2024-11-27T12:28:37,763 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1. 2024-11-27T12:28:37,763 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1. after waiting 0 ms 2024-11-27T12:28:37,763 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1. 2024-11-27T12:28:37,763 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1. 2024-11-27T12:28:37,764 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1. 2024-11-27T12:28:37,764 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5bccd99aa8cdbafe7bc7d0490c78c0b1: Waiting for close lock at 1732710517763Disabling compacts and flushes for region at 1732710517763Disabling writes for close at 1732710517763Writing region close event to WAL at 1732710517763Closed at 1732710517764 (+1 ms) 2024-11-27T12:28:37,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741916_1094 (size=1623) 2024-11-27T12:28:37,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741916_1094 (size=1623) 2024-11-27T12:28:37,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741916_1094 (size=1623) 2024-11-27T12:28:37,781 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38485/hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679/wal.1732710517730, size=1.6 K (1623bytes) 2024-11-27T12:28:37,781 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38485/hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679/wal.1732710517730 2024-11-27T12:28:37,781 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38485/hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679/wal.1732710517730 after 0ms 2024-11-27T12:28:37,784 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38485/hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679/wal.1732710517730 took 4ms 2024-11-27T12:28:37,785 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38485/hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679/wal.1732710517730 so closing down 2024-11-27T12:28:37,786 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T12:28:37,787 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732710517730.temp 2024-11-27T12:28:37,788 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/recovered.edits/0000000000000000003-wal.1732710517730.temp 2024-11-27T12:28:37,788 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T12:28:37,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741917_1095 (size=1623) 2024-11-27T12:28:37,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741917_1095 (size=1623) 2024-11-27T12:28:37,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741917_1095 (size=1623) 2024-11-27T12:28:37,794 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/recovered.edits/0000000000000000003-wal.1732710517730.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-27T12:28:37,795 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/recovered.edits/0000000000000000003-wal.1732710517730.temp to hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/recovered.edits/0000000000000000012 2024-11-27T12:28:37,795 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 11 ms; skipped=0; WAL=hdfs://localhost:38485/hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679/wal.1732710517730, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-11-27T12:28:37,795 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38485/hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679/wal.1732710517730, journal: Splitting hdfs://localhost:38485/hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679/wal.1732710517730, size=1.6 K (1623bytes) at 1732710517781Finishing writing output for hdfs://localhost:38485/hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679/wal.1732710517730 so closing down at 1732710517786 (+5 ms)Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/recovered.edits/0000000000000000003-wal.1732710517730.temp at 1732710517788 (+2 ms)3 split writer threads finished at 1732710517788Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/recovered.edits/0000000000000000003-wal.1732710517730.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732710517794 (+6 ms)Rename recovered edits hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/recovered.edits/0000000000000000003-wal.1732710517730.temp to hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/recovered.edits/0000000000000000012 at 1732710517795 (+1 ms)Processed 10 edits across 1 Regions in 11 ms; skipped=0; WAL=hdfs://localhost:38485/hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679/wal.1732710517730, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1732710517795 2024-11-27T12:28:37,797 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38485/hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679/wal.1732710517730 to hdfs://localhost:38485/hbase/oldWALs/wal.1732710517730 2024-11-27T12:28:37,798 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/recovered.edits/0000000000000000012 2024-11-27T12:28:37,849 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T12:28:38,103 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:38,105 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:38,118 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679/wal.1732710518106, exclude list is [], retry=0 2024-11-27T12:28:38,121 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:38,121 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:38,121 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:38,122 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679/wal.1732710518106 2024-11-27T12:28:38,124 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:34427:34427)] 2024-11-27T12:28:38,124 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5bccd99aa8cdbafe7bc7d0490c78c0b1, NAME => 'testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:38,124 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:38,124 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:38,124 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:38,128 INFO [StoreOpener-5bccd99aa8cdbafe7bc7d0490c78c0b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:38,130 INFO [StoreOpener-5bccd99aa8cdbafe7bc7d0490c78c0b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bccd99aa8cdbafe7bc7d0490c78c0b1 columnFamilyName a 2024-11-27T12:28:38,130 DEBUG [StoreOpener-5bccd99aa8cdbafe7bc7d0490c78c0b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:38,131 INFO [StoreOpener-5bccd99aa8cdbafe7bc7d0490c78c0b1-1 {}] regionserver.HStore(327): Store=5bccd99aa8cdbafe7bc7d0490c78c0b1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:38,132 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:38,132 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:38,134 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:38,134 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/recovered.edits/0000000000000000012 2024-11-27T12:28:38,137 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/recovered.edits/0000000000000000012 2024-11-27T12:28:38,137 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5bccd99aa8cdbafe7bc7d0490c78c0b1 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-27T12:28:38,154 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/.tmp/a/177d45f87d5442278235f99f676862a4 is 79, key is testDatalossWhenInputError/a:x0/1732710517754/Put/seqid=0 2024-11-27T12:28:38,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741919_1097 (size=5808) 2024-11-27T12:28:38,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741919_1097 (size=5808) 2024-11-27T12:28:38,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741919_1097 (size=5808) 2024-11-27T12:28:38,161 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/.tmp/a/177d45f87d5442278235f99f676862a4 2024-11-27T12:28:38,172 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/.tmp/a/177d45f87d5442278235f99f676862a4 as hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/a/177d45f87d5442278235f99f676862a4 2024-11-27T12:28:38,182 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/a/177d45f87d5442278235f99f676862a4, entries=10, sequenceid=12, filesize=5.7 K 2024-11-27T12:28:38,183 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 5bccd99aa8cdbafe7bc7d0490c78c0b1 in 46ms, sequenceid=12, compaction requested=false; wal=null 2024-11-27T12:28:38,183 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/recovered.edits/0000000000000000012 2024-11-27T12:28:38,185 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:38,185 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:38,188 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:38,190 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-27T12:28:38,191 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5bccd99aa8cdbafe7bc7d0490c78c0b1; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59454469, jitterRate=-0.11405937373638153}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T12:28:38,192 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5bccd99aa8cdbafe7bc7d0490c78c0b1: Writing region info on filesystem at 1732710518125Initializing all the Stores at 1732710518127 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710518127Obtaining lock to block concurrent updates at 1732710518137 (+10 ms)Preparing flush snapshotting stores in 5bccd99aa8cdbafe7bc7d0490c78c0b1 at 1732710518137Finished memstore snapshotting testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732710518137Flushing stores of testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1. at 1732710518137Flushing 5bccd99aa8cdbafe7bc7d0490c78c0b1/a: creating writer at 1732710518137Flushing 5bccd99aa8cdbafe7bc7d0490c78c0b1/a: appending metadata at 1732710518154 (+17 ms)Flushing 5bccd99aa8cdbafe7bc7d0490c78c0b1/a: closing flushed file at 1732710518154Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d7f1147: reopening flushed file at 1732710518170 (+16 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 5bccd99aa8cdbafe7bc7d0490c78c0b1 in 46ms, sequenceid=12, compaction requested=false; wal=null at 1732710518183 (+13 ms)Cleaning up temporary data from old regions at 1732710518185 (+2 ms)Region opened successfully at 1732710518192 (+7 ms) 2024-11-27T12:28:38,195 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5bccd99aa8cdbafe7bc7d0490c78c0b1, NAME => 'testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:38,195 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732710517680.5bccd99aa8cdbafe7bc7d0490c78c0b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:38,195 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:38,195 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:38,196 INFO [StoreOpener-5bccd99aa8cdbafe7bc7d0490c78c0b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:38,197 INFO [StoreOpener-5bccd99aa8cdbafe7bc7d0490c78c0b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bccd99aa8cdbafe7bc7d0490c78c0b1 columnFamilyName a 2024-11-27T12:28:38,197 DEBUG [StoreOpener-5bccd99aa8cdbafe7bc7d0490c78c0b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:38,202 DEBUG [StoreOpener-5bccd99aa8cdbafe7bc7d0490c78c0b1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/a/177d45f87d5442278235f99f676862a4 2024-11-27T12:28:38,202 INFO [StoreOpener-5bccd99aa8cdbafe7bc7d0490c78c0b1-1 {}] regionserver.HStore(327): Store=5bccd99aa8cdbafe7bc7d0490c78c0b1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:38,202 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:38,203 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:38,204 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:38,204 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:38,204 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:38,205 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5bccd99aa8cdbafe7bc7d0490c78c0b1 2024-11-27T12:28:38,207 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testDatalossWhenInputError/5bccd99aa8cdbafe7bc7d0490c78c0b1/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-27T12:28:38,208 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5bccd99aa8cdbafe7bc7d0490c78c0b1; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74779711, jitterRate=0.11430452764034271}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T12:28:38,208 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5bccd99aa8cdbafe7bc7d0490c78c0b1: Writing region info on filesystem at 1732710518195Initializing all the Stores at 1732710518196 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710518196Cleaning up temporary data from old regions at 1732710518204 (+8 ms)Region opened successfully at 1732710518208 (+4 ms) 2024-11-27T12:28:38,224 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=431 (was 421) Potentially hanging thread: PacketResponder: BP-39017082-172.17.0.3-1732710484128:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-39017082-172.17.0.3-1732710484128:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:49822 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:56046 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:55074 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:49862 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:55968 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-39017082-172.17.0.3-1732710484128:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:55036 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=239 (was 239), ProcessCount=11 (was 11), AvailableMemoryMB=8760 (was 8770) 2024-11-27T12:28:38,225 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-27T12:28:38,238 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=431, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=239, ProcessCount=11, AvailableMemoryMB=8760 2024-11-27T12:28:38,238 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-27T12:28:38,251 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:38,253 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:38,253 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:28:38,256 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-55763505, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/hregion-55763505, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:38,268 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-55763505/hregion-55763505.1732710518256, exclude list is [], retry=0 2024-11-27T12:28:38,271 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:38,271 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:38,272 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:38,278 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-55763505/hregion-55763505.1732710518256 2024-11-27T12:28:38,278 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:34427:34427)] 2024-11-27T12:28:38,278 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f1454dd207fba3b2b6d8fbc99f03634c, NAME => 'testCompactedBulkLoadedFiles,,1732710518252.f1454dd207fba3b2b6d8fbc99f03634c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38485/hbase 2024-11-27T12:28:38,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741921_1099 (size=63) 2024-11-27T12:28:38,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741921_1099 (size=63) 2024-11-27T12:28:38,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741921_1099 (size=63) 2024-11-27T12:28:38,287 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732710518252.f1454dd207fba3b2b6d8fbc99f03634c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:38,288 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,290 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1454dd207fba3b2b6d8fbc99f03634c columnFamilyName a 2024-11-27T12:28:38,290 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:38,290 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.HStore(327): Store=f1454dd207fba3b2b6d8fbc99f03634c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:38,290 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,291 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1454dd207fba3b2b6d8fbc99f03634c columnFamilyName b 2024-11-27T12:28:38,292 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:38,292 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.HStore(327): Store=f1454dd207fba3b2b6d8fbc99f03634c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:38,292 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,293 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1454dd207fba3b2b6d8fbc99f03634c columnFamilyName c 2024-11-27T12:28:38,293 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:38,294 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.HStore(327): Store=f1454dd207fba3b2b6d8fbc99f03634c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:38,294 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,295 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,295 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,296 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,296 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,296 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T12:28:38,298 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,300 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:28:38,300 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f1454dd207fba3b2b6d8fbc99f03634c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70699707, jitterRate=0.0535077303647995}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T12:28:38,301 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f1454dd207fba3b2b6d8fbc99f03634c: Writing region info on filesystem at 1732710518287Initializing all the Stores at 1732710518288 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710518288Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710518288Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710518288Cleaning up temporary data from old regions at 1732710518296 (+8 ms)Region opened successfully at 1732710518301 (+5 ms) 2024-11-27T12:28:38,301 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f1454dd207fba3b2b6d8fbc99f03634c, disabling compactions & flushes 2024-11-27T12:28:38,301 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732710518252.f1454dd207fba3b2b6d8fbc99f03634c. 2024-11-27T12:28:38,301 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732710518252.f1454dd207fba3b2b6d8fbc99f03634c. 2024-11-27T12:28:38,301 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732710518252.f1454dd207fba3b2b6d8fbc99f03634c. after waiting 0 ms 2024-11-27T12:28:38,301 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732710518252.f1454dd207fba3b2b6d8fbc99f03634c. 2024-11-27T12:28:38,301 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732710518252.f1454dd207fba3b2b6d8fbc99f03634c. 2024-11-27T12:28:38,301 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f1454dd207fba3b2b6d8fbc99f03634c: Waiting for close lock at 1732710518301Disabling compacts and flushes for region at 1732710518301Disabling writes for close at 1732710518301Writing region close event to WAL at 1732710518301Closed at 1732710518301 2024-11-27T12:28:38,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741920_1098 (size=93) 2024-11-27T12:28:38,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741920_1098 (size=93) 2024-11-27T12:28:38,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741920_1098 (size=93) 2024-11-27T12:28:38,306 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T12:28:38,306 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-55763505:(num 1732710518256) 2024-11-27T12:28:38,306 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:38,308 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:38,323 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710518308, exclude list is [], retry=0 2024-11-27T12:28:38,325 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:38,326 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:38,326 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:38,328 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710518308 2024-11-27T12:28:38,328 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:45591:45591)] 2024-11-27T12:28:38,328 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f1454dd207fba3b2b6d8fbc99f03634c, NAME => 'testCompactedBulkLoadedFiles,,1732710518252.f1454dd207fba3b2b6d8fbc99f03634c.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:38,328 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732710518252.f1454dd207fba3b2b6d8fbc99f03634c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:38,328 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,329 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,330 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,331 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1454dd207fba3b2b6d8fbc99f03634c columnFamilyName a 2024-11-27T12:28:38,331 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:38,332 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.HStore(327): Store=f1454dd207fba3b2b6d8fbc99f03634c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:38,332 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,333 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1454dd207fba3b2b6d8fbc99f03634c columnFamilyName b 2024-11-27T12:28:38,333 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:38,334 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.HStore(327): Store=f1454dd207fba3b2b6d8fbc99f03634c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:38,334 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,335 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1454dd207fba3b2b6d8fbc99f03634c columnFamilyName c 2024-11-27T12:28:38,335 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:38,336 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.HStore(327): Store=f1454dd207fba3b2b6d8fbc99f03634c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:38,336 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,337 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,338 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,339 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,339 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,339 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T12:28:38,341 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:38,341 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f1454dd207fba3b2b6d8fbc99f03634c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60226068, jitterRate=-0.10256165266036987}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T12:28:38,342 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f1454dd207fba3b2b6d8fbc99f03634c: Writing region info on filesystem at 1732710518329Initializing all the Stores at 1732710518330 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710518330Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710518330Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710518330Cleaning up temporary data from old regions at 1732710518339 (+9 ms)Region opened successfully at 1732710518342 (+3 ms) 2024-11-27T12:28:38,345 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732710518345/Put/seqid=0 2024-11-27T12:28:38,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741923_1101 (size=4875) 2024-11-27T12:28:38,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741923_1101 (size=4875) 2024-11-27T12:28:38,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741923_1101 (size=4875) 2024-11-27T12:28:38,354 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732710518354/Put/seqid=0 2024-11-27T12:28:38,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741924_1102 (size=4875) 2024-11-27T12:28:38,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741924_1102 (size=4875) 2024-11-27T12:28:38,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741924_1102 (size=4875) 2024-11-27T12:28:38,362 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732710518362/Put/seqid=0 2024-11-27T12:28:38,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741925_1103 (size=4875) 2024-11-27T12:28:38,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741925_1103 (size=4875) 2024-11-27T12:28:38,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741925_1103 (size=4875) 2024-11-27T12:28:38,368 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38485/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in f1454dd207fba3b2b6d8fbc99f03634c/a 2024-11-27T12:28:38,372 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-27T12:28:38,373 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-27T12:28:38,373 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38485/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in f1454dd207fba3b2b6d8fbc99f03634c/a 2024-11-27T12:28:38,377 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-27T12:28:38,377 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-27T12:28:38,377 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38485/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in f1454dd207fba3b2b6d8fbc99f03634c/a 2024-11-27T12:28:38,381 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-27T12:28:38,381 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-27T12:28:38,381 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f1454dd207fba3b2b6d8fbc99f03634c 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-27T12:28:38,397 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/.tmp/a/aea57614ec7a42a9ab9a441efdf5ec41 is 55, key is testCompactedBulkLoadedFiles/a:a/1732710518342/Put/seqid=0 2024-11-27T12:28:38,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741926_1104 (size=5107) 2024-11-27T12:28:38,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741926_1104 (size=5107) 2024-11-27T12:28:38,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741926_1104 (size=5107) 2024-11-27T12:28:38,403 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/.tmp/a/aea57614ec7a42a9ab9a441efdf5ec41 2024-11-27T12:28:38,408 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/.tmp/a/aea57614ec7a42a9ab9a441efdf5ec41 as hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/aea57614ec7a42a9ab9a441efdf5ec41 2024-11-27T12:28:38,413 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/aea57614ec7a42a9ab9a441efdf5ec41, entries=1, sequenceid=4, filesize=5.0 K 2024-11-27T12:28:38,414 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for f1454dd207fba3b2b6d8fbc99f03634c in 33ms, sequenceid=4, compaction requested=false 2024-11-27T12:28:38,414 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f1454dd207fba3b2b6d8fbc99f03634c: 2024-11-27T12:28:38,416 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/166c8ca3157c48b0bc65f5cccf8a4ad5_SeqId_4_ 2024-11-27T12:28:38,417 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/6e5d26ece4ea4bc99b8bcb119d3ad872_SeqId_4_ 2024-11-27T12:28:38,417 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/c4f31ddb6ea74690b25bab95a0924be0_SeqId_4_ 2024-11-27T12:28:38,418 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38485/hbase/testCompactedBulkLoadedFiles/hfile0 into f1454dd207fba3b2b6d8fbc99f03634c/a as hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/166c8ca3157c48b0bc65f5cccf8a4ad5_SeqId_4_ - updating store file list. 2024-11-27T12:28:38,422 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 166c8ca3157c48b0bc65f5cccf8a4ad5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T12:28:38,422 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/166c8ca3157c48b0bc65f5cccf8a4ad5_SeqId_4_ into f1454dd207fba3b2b6d8fbc99f03634c/a 2024-11-27T12:28:38,422 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38485/hbase/testCompactedBulkLoadedFiles/hfile0 into f1454dd207fba3b2b6d8fbc99f03634c/a (new location: hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/166c8ca3157c48b0bc65f5cccf8a4ad5_SeqId_4_) 2024-11-27T12:28:38,423 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38485/hbase/testCompactedBulkLoadedFiles/hfile1 into f1454dd207fba3b2b6d8fbc99f03634c/a as hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/6e5d26ece4ea4bc99b8bcb119d3ad872_SeqId_4_ - updating store file list. 2024-11-27T12:28:38,427 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6e5d26ece4ea4bc99b8bcb119d3ad872_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T12:28:38,428 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/6e5d26ece4ea4bc99b8bcb119d3ad872_SeqId_4_ into f1454dd207fba3b2b6d8fbc99f03634c/a 2024-11-27T12:28:38,428 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38485/hbase/testCompactedBulkLoadedFiles/hfile1 into f1454dd207fba3b2b6d8fbc99f03634c/a (new location: hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/6e5d26ece4ea4bc99b8bcb119d3ad872_SeqId_4_) 2024-11-27T12:28:38,429 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38485/hbase/testCompactedBulkLoadedFiles/hfile2 into f1454dd207fba3b2b6d8fbc99f03634c/a as hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/c4f31ddb6ea74690b25bab95a0924be0_SeqId_4_ - updating store file list. 2024-11-27T12:28:38,433 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for c4f31ddb6ea74690b25bab95a0924be0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T12:28:38,433 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/c4f31ddb6ea74690b25bab95a0924be0_SeqId_4_ into f1454dd207fba3b2b6d8fbc99f03634c/a 2024-11-27T12:28:38,433 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38485/hbase/testCompactedBulkLoadedFiles/hfile2 into f1454dd207fba3b2b6d8fbc99f03634c/a (new location: hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/c4f31ddb6ea74690b25bab95a0924be0_SeqId_4_) 2024-11-27T12:28:38,440 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:28:38,440 DEBUG [Time-limited test {}] regionserver.HStore(1541): f1454dd207fba3b2b6d8fbc99f03634c/a is initiating major compaction (all files) 2024-11-27T12:28:38,440 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of f1454dd207fba3b2b6d8fbc99f03634c/a in testCompactedBulkLoadedFiles,,1732710518252.f1454dd207fba3b2b6d8fbc99f03634c. 2024-11-27T12:28:38,440 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/aea57614ec7a42a9ab9a441efdf5ec41, hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/166c8ca3157c48b0bc65f5cccf8a4ad5_SeqId_4_, hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/6e5d26ece4ea4bc99b8bcb119d3ad872_SeqId_4_, hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/c4f31ddb6ea74690b25bab95a0924be0_SeqId_4_] into tmpdir=hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/.tmp, totalSize=19.3 K 2024-11-27T12:28:38,441 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting aea57614ec7a42a9ab9a441efdf5ec41, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732710518342 2024-11-27T12:28:38,441 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 166c8ca3157c48b0bc65f5cccf8a4ad5_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-27T12:28:38,441 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 6e5d26ece4ea4bc99b8bcb119d3ad872_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-27T12:28:38,442 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c4f31ddb6ea74690b25bab95a0924be0_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-27T12:28:38,453 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/.tmp/a/7bf2549136ca43e7aab902a7176ea323 is 55, key is testCompactedBulkLoadedFiles/a:a/1732710518342/Put/seqid=0 2024-11-27T12:28:38,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741927_1105 (size=6154) 2024-11-27T12:28:38,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741927_1105 (size=6154) 2024-11-27T12:28:38,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741927_1105 (size=6154) 2024-11-27T12:28:38,518 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/.tmp/a/7bf2549136ca43e7aab902a7176ea323 as hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/7bf2549136ca43e7aab902a7176ea323 2024-11-27T12:28:38,527 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in f1454dd207fba3b2b6d8fbc99f03634c/a of f1454dd207fba3b2b6d8fbc99f03634c into 7bf2549136ca43e7aab902a7176ea323(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:28:38,527 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for f1454dd207fba3b2b6d8fbc99f03634c: 2024-11-27T12:28:38,527 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-27T12:28:38,527 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-27T12:28:38,559 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38485/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710518308, size=0 (0bytes) 2024-11-27T12:28:38,559 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38485/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710518308 might be still open, length is 0 2024-11-27T12:28:38,559 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38485/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710518308 2024-11-27T12:28:38,559 WARN [IPC Server handler 4 on default port 38485 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710518308 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-27T12:28:38,559 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38485/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710518308 after 0ms 2024-11-27T12:28:39,060 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-27T12:28:39,060 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-27T12:28:39,061 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-27T12:28:39,061 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-27T12:28:40,761 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:49890 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:38877:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49890 dst: /127.0.0.1:38877 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38877 remote=/127.0.0.1:49890]. Total timeout mills is 60000, 57766 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:40,762 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:56080 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:38365:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56080 dst: /127.0.0.1:38365 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:40,762 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:55116 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:37469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55116 dst: /127.0.0.1:37469 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:40,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741922_1106 (size=1659) 2024-11-27T12:28:40,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741922_1106 (size=1659) 2024-11-27T12:28:42,560 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38485/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710518308 after 4001ms 2024-11-27T12:28:42,563 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38485/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710518308 took 4004ms 2024-11-27T12:28:42,565 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38485/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710518308; continuing. 2024-11-27T12:28:42,565 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38485/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710518308 so closing down 2024-11-27T12:28:42,565 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T12:28:42,566 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732710518308.temp 2024-11-27T12:28:42,567 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/recovered.edits/0000000000000000003-wal.1732710518308.temp 2024-11-27T12:28:42,567 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T12:28:42,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741928_1107 (size=698) 2024-11-27T12:28:42,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741928_1107 (size=698) 2024-11-27T12:28:42,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741928_1107 (size=698) 2024-11-27T12:28:42,576 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/recovered.edits/0000000000000000003-wal.1732710518308.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-27T12:28:42,577 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/recovered.edits/0000000000000000003-wal.1732710518308.temp to hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/recovered.edits/0000000000000000008 2024-11-27T12:28:42,577 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:38485/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710518308, size=0, length=0, corrupted=false, cancelled=false 2024-11-27T12:28:42,577 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38485/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710518308, journal: Splitting hdfs://localhost:38485/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710518308, size=0 (0bytes) at 1732710518559Finishing writing output for hdfs://localhost:38485/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710518308 so closing down at 1732710522565 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/recovered.edits/0000000000000000003-wal.1732710518308.temp at 1732710522567 (+2 ms)3 split writer threads finished at 1732710522567Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/recovered.edits/0000000000000000003-wal.1732710518308.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732710522576 (+9 ms)Rename recovered edits hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/recovered.edits/0000000000000000003-wal.1732710518308.temp to hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/recovered.edits/0000000000000000008 at 1732710522577 (+1 ms)Processed 5 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:38485/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710518308, size=0, length=0, corrupted=false, cancelled=false at 1732710522577 2024-11-27T12:28:42,579 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38485/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710518308 to hdfs://localhost:38485/hbase/oldWALs/wal.1732710518308 2024-11-27T12:28:42,580 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/recovered.edits/0000000000000000008 2024-11-27T12:28:42,580 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:42,581 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:42,595 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710522582, exclude list is [], retry=0 2024-11-27T12:28:42,597 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:42,597 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:42,597 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:42,599 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710522582 2024-11-27T12:28:42,599 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:34427:34427)] 2024-11-27T12:28:42,599 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f1454dd207fba3b2b6d8fbc99f03634c, NAME => 'testCompactedBulkLoadedFiles,,1732710518252.f1454dd207fba3b2b6d8fbc99f03634c.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:42,600 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732710518252.f1454dd207fba3b2b6d8fbc99f03634c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:42,600 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:42,600 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:42,601 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:42,602 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1454dd207fba3b2b6d8fbc99f03634c columnFamilyName a 2024-11-27T12:28:42,602 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:42,608 DEBUG [StoreFileOpener-f1454dd207fba3b2b6d8fbc99f03634c-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 166c8ca3157c48b0bc65f5cccf8a4ad5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T12:28:42,608 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/166c8ca3157c48b0bc65f5cccf8a4ad5_SeqId_4_ 2024-11-27T12:28:42,612 DEBUG [StoreFileOpener-f1454dd207fba3b2b6d8fbc99f03634c-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6e5d26ece4ea4bc99b8bcb119d3ad872_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T12:28:42,612 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/6e5d26ece4ea4bc99b8bcb119d3ad872_SeqId_4_ 2024-11-27T12:28:42,616 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/7bf2549136ca43e7aab902a7176ea323 2024-11-27T12:28:42,620 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/aea57614ec7a42a9ab9a441efdf5ec41 2024-11-27T12:28:42,624 DEBUG [StoreFileOpener-f1454dd207fba3b2b6d8fbc99f03634c-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for c4f31ddb6ea74690b25bab95a0924be0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T12:28:42,624 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/c4f31ddb6ea74690b25bab95a0924be0_SeqId_4_ 2024-11-27T12:28:42,624 WARN [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/166c8ca3157c48b0bc65f5cccf8a4ad5_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2b78ef1e 2024-11-27T12:28:42,624 WARN [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/6e5d26ece4ea4bc99b8bcb119d3ad872_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2b78ef1e 2024-11-27T12:28:42,624 WARN [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/aea57614ec7a42a9ab9a441efdf5ec41 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2b78ef1e 2024-11-27T12:28:42,624 WARN [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/c4f31ddb6ea74690b25bab95a0924be0_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2b78ef1e 2024-11-27T12:28:42,624 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/166c8ca3157c48b0bc65f5cccf8a4ad5_SeqId_4_, hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/6e5d26ece4ea4bc99b8bcb119d3ad872_SeqId_4_, hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/aea57614ec7a42a9ab9a441efdf5ec41, hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/c4f31ddb6ea74690b25bab95a0924be0_SeqId_4_] to archive 2024-11-27T12:28:42,626 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:28:42,627 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/166c8ca3157c48b0bc65f5cccf8a4ad5_SeqId_4_ to hdfs://localhost:38485/hbase/archive/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/166c8ca3157c48b0bc65f5cccf8a4ad5_SeqId_4_ 2024-11-27T12:28:42,628 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/6e5d26ece4ea4bc99b8bcb119d3ad872_SeqId_4_ to hdfs://localhost:38485/hbase/archive/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/6e5d26ece4ea4bc99b8bcb119d3ad872_SeqId_4_ 2024-11-27T12:28:42,629 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/aea57614ec7a42a9ab9a441efdf5ec41 to hdfs://localhost:38485/hbase/archive/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/aea57614ec7a42a9ab9a441efdf5ec41 2024-11-27T12:28:42,630 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/c4f31ddb6ea74690b25bab95a0924be0_SeqId_4_ to hdfs://localhost:38485/hbase/archive/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/a/c4f31ddb6ea74690b25bab95a0924be0_SeqId_4_ 2024-11-27T12:28:42,630 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.HStore(327): Store=f1454dd207fba3b2b6d8fbc99f03634c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:42,630 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:42,631 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1454dd207fba3b2b6d8fbc99f03634c columnFamilyName b 2024-11-27T12:28:42,631 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:42,631 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.HStore(327): Store=f1454dd207fba3b2b6d8fbc99f03634c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:42,631 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:42,632 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1454dd207fba3b2b6d8fbc99f03634c columnFamilyName c 2024-11-27T12:28:42,632 DEBUG [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:42,632 INFO [StoreOpener-f1454dd207fba3b2b6d8fbc99f03634c-1 {}] regionserver.HStore(327): Store=f1454dd207fba3b2b6d8fbc99f03634c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:42,632 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:42,633 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:42,635 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:42,635 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/recovered.edits/0000000000000000008 2024-11-27T12:28:42,639 DEBUG [Time-limited test {}] regionserver.HRegion(5836): f1454dd207fba3b2b6d8fbc99f03634c : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "f1454dd207fba3b2b6d8fbc99f03634c" family_name: "a" compaction_input: "aea57614ec7a42a9ab9a441efdf5ec41" compaction_input: "166c8ca3157c48b0bc65f5cccf8a4ad5_SeqId_4_" compaction_input: "6e5d26ece4ea4bc99b8bcb119d3ad872_SeqId_4_" compaction_input: "c4f31ddb6ea74690b25bab95a0924be0_SeqId_4_" compaction_output: "7bf2549136ca43e7aab902a7176ea323" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732710518252.f1454dd207fba3b2b6d8fbc99f03634c." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-27T12:28:42,639 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-27T12:28:42,639 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/recovered.edits/0000000000000000008 2024-11-27T12:28:42,640 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/recovered.edits/0000000000000000008 2024-11-27T12:28:42,641 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:42,641 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:42,642 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T12:28:42,643 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f1454dd207fba3b2b6d8fbc99f03634c 2024-11-27T12:28:42,645 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testCompactedBulkLoadedFiles/f1454dd207fba3b2b6d8fbc99f03634c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-27T12:28:42,646 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f1454dd207fba3b2b6d8fbc99f03634c; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75442556, jitterRate=0.12418168783187866}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T12:28:42,646 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f1454dd207fba3b2b6d8fbc99f03634c: Writing region info on filesystem at 1732710522600Initializing all the Stores at 1732710522601 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710522601Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710522601Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710522601Cleaning up temporary data from old regions at 1732710522641 (+40 ms)Region opened successfully at 1732710522646 (+5 ms) 2024-11-27T12:28:42,649 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f1454dd207fba3b2b6d8fbc99f03634c, disabling compactions & flushes 2024-11-27T12:28:42,649 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732710518252.f1454dd207fba3b2b6d8fbc99f03634c. 2024-11-27T12:28:42,649 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732710518252.f1454dd207fba3b2b6d8fbc99f03634c. 2024-11-27T12:28:42,649 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732710518252.f1454dd207fba3b2b6d8fbc99f03634c. after waiting 0 ms 2024-11-27T12:28:42,649 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732710518252.f1454dd207fba3b2b6d8fbc99f03634c. 2024-11-27T12:28:42,649 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732710518252.f1454dd207fba3b2b6d8fbc99f03634c. 2024-11-27T12:28:42,649 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f1454dd207fba3b2b6d8fbc99f03634c: Waiting for close lock at 1732710522648Disabling compacts and flushes for region at 1732710522648Disabling writes for close at 1732710522649 (+1 ms)Writing region close event to WAL at 1732710522649Closed at 1732710522649 2024-11-27T12:28:42,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741929_1108 (size=93) 2024-11-27T12:28:42,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741929_1108 (size=93) 2024-11-27T12:28:42,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741929_1108 (size=93) 2024-11-27T12:28:42,654 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T12:28:42,654 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732710522582) 2024-11-27T12:28:42,667 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=443 (was 431) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1771064945_22 at /127.0.0.1:53702 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:38485 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1612229954) connection to localhost/127.0.0.1:38485 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1771064945_22 at /127.0.0.1:33170 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1771064945_22 at /127.0.0.1:50252 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=220 (was 239), ProcessCount=11 (was 11), AvailableMemoryMB=8756 (was 8760) 2024-11-27T12:28:42,667 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-27T12:28:42,679 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=443, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=220, ProcessCount=11, AvailableMemoryMB=8755 2024-11-27T12:28:42,679 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-27T12:28:42,693 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:42,695 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:28:42,696 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:28:42,698 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-38861066, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/hregion-38861066, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:42,711 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-38861066/hregion-38861066.1732710522698, exclude list is [], retry=0 2024-11-27T12:28:42,713 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:42,713 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:42,714 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:42,716 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-38861066/hregion-38861066.1732710522698 2024-11-27T12:28:42,718 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:46317:46317)] 2024-11-27T12:28:42,718 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 47ce95c5f343a8652111d54dc4636612, NAME => 'testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38485/hbase 2024-11-27T12:28:42,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741931_1110 (size=67) 2024-11-27T12:28:42,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741931_1110 (size=67) 2024-11-27T12:28:42,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741931_1110 (size=67) 2024-11-27T12:28:42,729 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:42,730 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,731 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ce95c5f343a8652111d54dc4636612 columnFamilyName a 2024-11-27T12:28:42,731 DEBUG [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:42,731 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(327): Store=47ce95c5f343a8652111d54dc4636612/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:42,731 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,732 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ce95c5f343a8652111d54dc4636612 columnFamilyName b 2024-11-27T12:28:42,733 DEBUG [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:42,733 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(327): Store=47ce95c5f343a8652111d54dc4636612/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:42,733 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,734 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ce95c5f343a8652111d54dc4636612 columnFamilyName c 2024-11-27T12:28:42,734 DEBUG [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:42,735 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(327): Store=47ce95c5f343a8652111d54dc4636612/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:42,735 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,735 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,736 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,737 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,737 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,737 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T12:28:42,738 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,740 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:28:42,740 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 47ce95c5f343a8652111d54dc4636612; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69471416, jitterRate=0.03520476818084717}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T12:28:42,741 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 47ce95c5f343a8652111d54dc4636612: Writing region info on filesystem at 1732710522729Initializing all the Stores at 1732710522729Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710522729Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710522729Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710522729Cleaning up temporary data from old regions at 1732710522737 (+8 ms)Region opened successfully at 1732710522741 (+4 ms) 2024-11-27T12:28:42,741 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 47ce95c5f343a8652111d54dc4636612, disabling compactions & flushes 2024-11-27T12:28:42,741 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612. 2024-11-27T12:28:42,741 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612. 2024-11-27T12:28:42,741 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612. after waiting 0 ms 2024-11-27T12:28:42,741 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612. 2024-11-27T12:28:42,741 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612. 2024-11-27T12:28:42,741 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 47ce95c5f343a8652111d54dc4636612: Waiting for close lock at 1732710522741Disabling compacts and flushes for region at 1732710522741Disabling writes for close at 1732710522741Writing region close event to WAL at 1732710522741Closed at 1732710522741 2024-11-27T12:28:42,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741930_1109 (size=93) 2024-11-27T12:28:42,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741930_1109 (size=93) 2024-11-27T12:28:42,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741930_1109 (size=93) 2024-11-27T12:28:42,746 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T12:28:42,746 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-38861066:(num 1732710522698) 2024-11-27T12:28:42,746 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:42,748 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:42,760 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522748, exclude list is [], retry=0 2024-11-27T12:28:42,762 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:42,763 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:42,763 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:42,765 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522748 2024-11-27T12:28:42,765 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:46317:46317),(127.0.0.1/127.0.0.1:34427:34427)] 2024-11-27T12:28:42,765 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 47ce95c5f343a8652111d54dc4636612, NAME => 'testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:42,765 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:42,766 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,766 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,767 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,768 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ce95c5f343a8652111d54dc4636612 columnFamilyName a 2024-11-27T12:28:42,768 DEBUG [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:42,768 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(327): Store=47ce95c5f343a8652111d54dc4636612/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:42,769 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,769 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ce95c5f343a8652111d54dc4636612 columnFamilyName b 2024-11-27T12:28:42,769 DEBUG [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:42,770 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(327): Store=47ce95c5f343a8652111d54dc4636612/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:42,770 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,770 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ce95c5f343a8652111d54dc4636612 columnFamilyName c 2024-11-27T12:28:42,771 DEBUG [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:42,771 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(327): Store=47ce95c5f343a8652111d54dc4636612/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:42,771 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,772 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,773 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,774 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,774 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,775 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T12:28:42,776 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,777 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 47ce95c5f343a8652111d54dc4636612; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72198822, jitterRate=0.07584628462791443}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T12:28:42,777 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 47ce95c5f343a8652111d54dc4636612: Writing region info on filesystem at 1732710522766Initializing all the Stores at 1732710522767 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710522767Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710522767Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710522767Cleaning up temporary data from old regions at 1732710522774 (+7 ms)Region opened successfully at 1732710522777 (+3 ms) 2024-11-27T12:28:42,784 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 47ce95c5f343a8652111d54dc4636612 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-27T12:28:42,799 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/.tmp/a/0c862f183296400099b132290fc66197 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732710522777/Put/seqid=0 2024-11-27T12:28:42,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741933_1112 (size=5958) 2024-11-27T12:28:42,806 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/.tmp/a/0c862f183296400099b132290fc66197 2024-11-27T12:28:42,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741933_1112 (size=5958) 2024-11-27T12:28:42,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741933_1112 (size=5958) 2024-11-27T12:28:42,812 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/.tmp/a/0c862f183296400099b132290fc66197 as hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/a/0c862f183296400099b132290fc66197 2024-11-27T12:28:42,817 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/a/0c862f183296400099b132290fc66197, entries=10, sequenceid=13, filesize=5.8 K 2024-11-27T12:28:42,819 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 47ce95c5f343a8652111d54dc4636612 in 35ms, sequenceid=13, compaction requested=false 2024-11-27T12:28:42,819 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 47ce95c5f343a8652111d54dc4636612: 2024-11-27T12:28:42,838 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 47ce95c5f343a8652111d54dc4636612, disabling compactions & flushes 2024-11-27T12:28:42,838 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612. 2024-11-27T12:28:42,838 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612. 2024-11-27T12:28:42,838 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612. after waiting 0 ms 2024-11-27T12:28:42,838 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612. 2024-11-27T12:28:42,839 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612. 2024-11-27T12:28:42,839 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612. 2024-11-27T12:28:42,839 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 47ce95c5f343a8652111d54dc4636612: Waiting for close lock at 1732710522838Disabling compacts and flushes for region at 1732710522838Disabling writes for close at 1732710522838Writing region close event to WAL at 1732710522839 (+1 ms)Closed at 1732710522839 2024-11-27T12:28:42,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741932_1111 (size=5869) 2024-11-27T12:28:42,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741932_1111 (size=5869) 2024-11-27T12:28:42,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741932_1111 (size=5869) 2024-11-27T12:28:42,857 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522748, size=5.7 K (5869bytes) 2024-11-27T12:28:42,857 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522748 2024-11-27T12:28:42,857 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522748 after 0ms 2024-11-27T12:28:42,860 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522748 took 3ms 2024-11-27T12:28:42,862 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522748 so closing down 2024-11-27T12:28:42,862 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T12:28:42,863 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732710522748.temp 2024-11-27T12:28:42,864 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000003-wal.1732710522748.temp 2024-11-27T12:28:42,864 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T12:28:42,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741934_1113 (size=5223) 2024-11-27T12:28:42,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741934_1113 (size=5223) 2024-11-27T12:28:42,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741934_1113 (size=5223) 2024-11-27T12:28:42,870 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000003-wal.1732710522748.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-27T12:28:42,872 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000003-wal.1732710522748.temp to hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000035 2024-11-27T12:28:42,872 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522748, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-11-27T12:28:42,872 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522748, journal: Splitting hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522748, size=5.7 K (5869bytes) at 1732710522857Finishing writing output for hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522748 so closing down at 1732710522862 (+5 ms)Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000003-wal.1732710522748.temp at 1732710522864 (+2 ms)3 split writer threads finished at 1732710522864Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000003-wal.1732710522748.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732710522870 (+6 ms)Rename recovered edits hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000003-wal.1732710522748.temp to hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000035 at 1732710522872 (+2 ms)Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522748, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1732710522872 2024-11-27T12:28:42,874 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522748 to hdfs://localhost:38485/hbase/oldWALs/wal.1732710522748 2024-11-27T12:28:42,874 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000035 2024-11-27T12:28:42,875 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:42,876 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:42,893 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522876, exclude list is [], retry=0 2024-11-27T12:28:42,896 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:42,896 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:42,896 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:42,898 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522876 2024-11-27T12:28:42,898 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:46317:46317)] 2024-11-27T12:28:42,899 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 47ce95c5f343a8652111d54dc4636612, NAME => 'testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:28:42,899 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:42,899 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,899 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,900 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,901 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ce95c5f343a8652111d54dc4636612 columnFamilyName a 2024-11-27T12:28:42,901 DEBUG [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:42,907 DEBUG [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/a/0c862f183296400099b132290fc66197 2024-11-27T12:28:42,907 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(327): Store=47ce95c5f343a8652111d54dc4636612/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:42,907 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,909 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ce95c5f343a8652111d54dc4636612 columnFamilyName b 2024-11-27T12:28:42,909 DEBUG [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:42,909 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(327): Store=47ce95c5f343a8652111d54dc4636612/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:42,909 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,910 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ce95c5f343a8652111d54dc4636612 columnFamilyName c 2024-11-27T12:28:42,910 DEBUG [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:42,911 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(327): Store=47ce95c5f343a8652111d54dc4636612/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:42,911 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,911 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,913 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,913 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000035 2024-11-27T12:28:42,916 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000035 2024-11-27T12:28:42,916 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 47ce95c5f343a8652111d54dc4636612 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-27T12:28:42,933 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/.tmp/b/a795f32a238d4a19860086fb694484dc is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732710522819/Put/seqid=0 2024-11-27T12:28:42,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741936_1115 (size=5958) 2024-11-27T12:28:42,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741936_1115 (size=5958) 2024-11-27T12:28:42,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741936_1115 (size=5958) 2024-11-27T12:28:42,940 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/.tmp/b/a795f32a238d4a19860086fb694484dc 2024-11-27T12:28:42,958 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/.tmp/c/eae682724233489ab5ff12f3a36b9a04 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732710522827/Put/seqid=0 2024-11-27T12:28:42,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741937_1116 (size=5958) 2024-11-27T12:28:42,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741937_1116 (size=5958) 2024-11-27T12:28:42,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741937_1116 (size=5958) 2024-11-27T12:28:42,965 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/.tmp/c/eae682724233489ab5ff12f3a36b9a04 2024-11-27T12:28:42,971 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/.tmp/b/a795f32a238d4a19860086fb694484dc as hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/b/a795f32a238d4a19860086fb694484dc 2024-11-27T12:28:42,976 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/b/a795f32a238d4a19860086fb694484dc, entries=10, sequenceid=35, filesize=5.8 K 2024-11-27T12:28:42,978 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/.tmp/c/eae682724233489ab5ff12f3a36b9a04 as hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/c/eae682724233489ab5ff12f3a36b9a04 2024-11-27T12:28:42,983 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/c/eae682724233489ab5ff12f3a36b9a04, entries=10, sequenceid=35, filesize=5.8 K 2024-11-27T12:28:42,983 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 47ce95c5f343a8652111d54dc4636612 in 67ms, sequenceid=35, compaction requested=false; wal=null 2024-11-27T12:28:42,984 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000035 2024-11-27T12:28:42,985 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,985 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,985 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T12:28:42,987 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:42,989 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-27T12:28:42,990 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 47ce95c5f343a8652111d54dc4636612; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73114737, jitterRate=0.0894944816827774}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T12:28:42,990 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 47ce95c5f343a8652111d54dc4636612: Writing region info on filesystem at 1732710522899Initializing all the Stores at 1732710522900 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710522900Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710522900Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710522900Obtaining lock to block concurrent updates at 1732710522916 (+16 ms)Preparing flush snapshotting stores in 47ce95c5f343a8652111d54dc4636612 at 1732710522916Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732710522916Flushing stores of testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612. at 1732710522917 (+1 ms)Flushing 47ce95c5f343a8652111d54dc4636612/b: creating writer at 1732710522917Flushing 47ce95c5f343a8652111d54dc4636612/b: appending metadata at 1732710522933 (+16 ms)Flushing 47ce95c5f343a8652111d54dc4636612/b: closing flushed file at 1732710522933Flushing 47ce95c5f343a8652111d54dc4636612/c: creating writer at 1732710522944 (+11 ms)Flushing 47ce95c5f343a8652111d54dc4636612/c: appending metadata at 1732710522958 (+14 ms)Flushing 47ce95c5f343a8652111d54dc4636612/c: closing flushed file at 1732710522958Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a330376: reopening flushed file at 1732710522970 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ad601cc: reopening flushed file at 1732710522977 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 47ce95c5f343a8652111d54dc4636612 in 67ms, sequenceid=35, compaction requested=false; wal=null at 1732710522983 (+6 ms)Cleaning up temporary data from old regions at 1732710522985 (+2 ms)Region opened successfully at 1732710522990 (+5 ms) 2024-11-27T12:28:42,997 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T12:28:43,057 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522876, size=0 (0bytes) 2024-11-27T12:28:43,057 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522876 might be still open, length is 0 2024-11-27T12:28:43,057 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522876 2024-11-27T12:28:43,058 WARN [IPC Server handler 2 on default port 38485 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522876 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-27T12:28:43,058 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522876 after 1ms 2024-11-27T12:28:43,759 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:33226 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:38365:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33226 dst: /127.0.0.1:38365 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38365 remote=/127.0.0.1:33226]. Total timeout mills is 60000, 59261 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:43,760 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:53768 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:38877:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53768 dst: /127.0.0.1:38877 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:43,760 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-862841147_22 at /127.0.0.1:50308 [Receiving block BP-39017082-172.17.0.3-1732710484128:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:37469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50308 dst: /127.0.0.1:37469 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:43,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741935_1117 (size=5215) 2024-11-27T12:28:43,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741935_1117 (size=5215) 2024-11-27T12:28:43,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741935_1117 (size=5215) 2024-11-27T12:28:47,059 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522876 after 4002ms 2024-11-27T12:28:47,068 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522876 took 4011ms 2024-11-27T12:28:47,070 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522876; continuing. 2024-11-27T12:28:47,070 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522876 so closing down 2024-11-27T12:28:47,070 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T12:28:47,072 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732710522876.temp 2024-11-27T12:28:47,073 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000037-wal.1732710522876.temp 2024-11-27T12:28:47,073 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T12:28:47,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741938_1118 (size=5223) 2024-11-27T12:28:47,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741938_1118 (size=5223) 2024-11-27T12:28:47,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741938_1118 (size=5223) 2024-11-27T12:28:47,083 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000037-wal.1732710522876.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-27T12:28:47,084 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000037-wal.1732710522876.temp to hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000066 2024-11-27T12:28:47,084 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522876, size=0, length=0, corrupted=false, cancelled=false 2024-11-27T12:28:47,084 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522876, journal: Splitting hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522876, size=0 (0bytes) at 1732710523057Finishing writing output for hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522876 so closing down at 1732710527070 (+4013 ms)Creating recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000037-wal.1732710522876.temp at 1732710527073 (+3 ms)3 split writer threads finished at 1732710527073Closed recovered edits writer path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000037-wal.1732710522876.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732710527083 (+10 ms)Rename recovered edits hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000037-wal.1732710522876.temp to hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000066 at 1732710527084 (+1 ms)Processed 30 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522876, size=0, length=0, corrupted=false, cancelled=false at 1732710527084 2024-11-27T12:28:47,085 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522876 to hdfs://localhost:38485/hbase/oldWALs/wal.1732710522876 2024-11-27T12:28:47,086 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000066 2024-11-27T12:28:47,086 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T12:28:47,087 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38485/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693, archiveDir=hdfs://localhost:38485/hbase/oldWALs, maxLogs=32 2024-11-27T12:28:47,099 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710527087, exclude list is [], retry=0 2024-11-27T12:28:47,101 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38365,DS-a9c334cc-4027-492e-8f93-da489a14b1de,DISK] 2024-11-27T12:28:47,102 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37469,DS-aa5528be-d81f-4bfd-bd99-7be23f3c7ddf,DISK] 2024-11-27T12:28:47,102 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38877,DS-d7bfcce3-e06d-4841-9b29-b5f85387a4c5,DISK] 2024-11-27T12:28:47,103 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710527087 2024-11-27T12:28:47,104 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34427:34427),(127.0.0.1/127.0.0.1:45591:45591),(127.0.0.1/127.0.0.1:46317:46317)] 2024-11-27T12:28:47,105 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:28:47,106 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:47,107 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ce95c5f343a8652111d54dc4636612 columnFamilyName a 2024-11-27T12:28:47,107 DEBUG [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:47,111 DEBUG [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/a/0c862f183296400099b132290fc66197 2024-11-27T12:28:47,112 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(327): Store=47ce95c5f343a8652111d54dc4636612/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:47,112 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:47,113 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ce95c5f343a8652111d54dc4636612 columnFamilyName b 2024-11-27T12:28:47,113 DEBUG [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:47,117 DEBUG [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/b/a795f32a238d4a19860086fb694484dc 2024-11-27T12:28:47,117 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(327): Store=47ce95c5f343a8652111d54dc4636612/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:47,118 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:47,118 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ce95c5f343a8652111d54dc4636612 columnFamilyName c 2024-11-27T12:28:47,118 DEBUG [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:28:47,123 DEBUG [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/c/eae682724233489ab5ff12f3a36b9a04 2024-11-27T12:28:47,123 INFO [StoreOpener-47ce95c5f343a8652111d54dc4636612-1 {}] regionserver.HStore(327): Store=47ce95c5f343a8652111d54dc4636612/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:28:47,123 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:47,124 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:47,125 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:47,126 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000066 2024-11-27T12:28:47,131 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000066 2024-11-27T12:28:47,131 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 47ce95c5f343a8652111d54dc4636612 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-27T12:28:47,145 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/.tmp/a/b6feaf5ebc8f443baa670f77d6d3b001 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732710522997/Put/seqid=0 2024-11-27T12:28:47,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741940_1120 (size=5958) 2024-11-27T12:28:47,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741940_1120 (size=5958) 2024-11-27T12:28:47,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741940_1120 (size=5958) 2024-11-27T12:28:47,153 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/.tmp/a/b6feaf5ebc8f443baa670f77d6d3b001 2024-11-27T12:28:47,175 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/.tmp/b/92abc86ba6af4a4492125e5a6d0e9583 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732710523005/Put/seqid=0 2024-11-27T12:28:47,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741941_1121 (size=5958) 2024-11-27T12:28:47,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741941_1121 (size=5958) 2024-11-27T12:28:47,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741941_1121 (size=5958) 2024-11-27T12:28:47,182 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/.tmp/b/92abc86ba6af4a4492125e5a6d0e9583 2024-11-27T12:28:47,200 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/.tmp/c/8a03dac952f94860ba65dfca44295efb is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732710523012/Put/seqid=0 2024-11-27T12:28:47,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741942_1122 (size=5958) 2024-11-27T12:28:47,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741942_1122 (size=5958) 2024-11-27T12:28:47,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741942_1122 (size=5958) 2024-11-27T12:28:47,207 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/.tmp/c/8a03dac952f94860ba65dfca44295efb 2024-11-27T12:28:47,213 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/.tmp/a/b6feaf5ebc8f443baa670f77d6d3b001 as hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/a/b6feaf5ebc8f443baa670f77d6d3b001 2024-11-27T12:28:47,218 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/a/b6feaf5ebc8f443baa670f77d6d3b001, entries=10, sequenceid=66, filesize=5.8 K 2024-11-27T12:28:47,220 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/.tmp/b/92abc86ba6af4a4492125e5a6d0e9583 as hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/b/92abc86ba6af4a4492125e5a6d0e9583 2024-11-27T12:28:47,225 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/b/92abc86ba6af4a4492125e5a6d0e9583, entries=10, sequenceid=66, filesize=5.8 K 2024-11-27T12:28:47,226 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/.tmp/c/8a03dac952f94860ba65dfca44295efb as hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/c/8a03dac952f94860ba65dfca44295efb 2024-11-27T12:28:47,231 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/c/8a03dac952f94860ba65dfca44295efb, entries=10, sequenceid=66, filesize=5.8 K 2024-11-27T12:28:47,232 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 47ce95c5f343a8652111d54dc4636612 in 101ms, sequenceid=66, compaction requested=false; wal=null 2024-11-27T12:28:47,233 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/0000000000000000066 2024-11-27T12:28:47,234 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:47,234 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:47,234 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T12:28:47,236 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 47ce95c5f343a8652111d54dc4636612 2024-11-27T12:28:47,238 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/hbase/data/default/testReplayEditsWrittenViaHRegion/47ce95c5f343a8652111d54dc4636612/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-27T12:28:47,239 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 47ce95c5f343a8652111d54dc4636612; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74623806, jitterRate=0.1119813621044159}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T12:28:47,239 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 47ce95c5f343a8652111d54dc4636612: Writing region info on filesystem at 1732710527105Initializing all the Stores at 1732710527106 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710527106Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710527106Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710527106Obtaining lock to block concurrent updates at 1732710527131 (+25 ms)Preparing flush snapshotting stores in 47ce95c5f343a8652111d54dc4636612 at 1732710527131Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732710527131Flushing stores of testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612. at 1732710527131Flushing 47ce95c5f343a8652111d54dc4636612/a: creating writer at 1732710527131Flushing 47ce95c5f343a8652111d54dc4636612/a: appending metadata at 1732710527144 (+13 ms)Flushing 47ce95c5f343a8652111d54dc4636612/a: closing flushed file at 1732710527144Flushing 47ce95c5f343a8652111d54dc4636612/b: creating writer at 1732710527157 (+13 ms)Flushing 47ce95c5f343a8652111d54dc4636612/b: appending metadata at 1732710527174 (+17 ms)Flushing 47ce95c5f343a8652111d54dc4636612/b: closing flushed file at 1732710527174Flushing 47ce95c5f343a8652111d54dc4636612/c: creating writer at 1732710527185 (+11 ms)Flushing 47ce95c5f343a8652111d54dc4636612/c: appending metadata at 1732710527199 (+14 ms)Flushing 47ce95c5f343a8652111d54dc4636612/c: closing flushed file at 1732710527199Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51e1c56f: reopening flushed file at 1732710527212 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@599059d2: reopening flushed file at 1732710527219 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@948f136: reopening flushed file at 1732710527225 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 47ce95c5f343a8652111d54dc4636612 in 101ms, sequenceid=66, compaction requested=false; wal=null at 1732710527232 (+7 ms)Cleaning up temporary data from old regions at 1732710527234 (+2 ms)Region opened successfully at 1732710527239 (+5 ms) 2024-11-27T12:28:47,253 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 47ce95c5f343a8652111d54dc4636612, disabling compactions & flushes 2024-11-27T12:28:47,253 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612. 2024-11-27T12:28:47,253 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612. 2024-11-27T12:28:47,253 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612. after waiting 0 ms 2024-11-27T12:28:47,253 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612. 2024-11-27T12:28:47,255 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732710522694.47ce95c5f343a8652111d54dc4636612. 2024-11-27T12:28:47,255 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 47ce95c5f343a8652111d54dc4636612: Waiting for close lock at 1732710527253Disabling compacts and flushes for region at 1732710527253Disabling writes for close at 1732710527253Writing region close event to WAL at 1732710527255 (+2 ms)Closed at 1732710527255 2024-11-27T12:28:47,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741939_1119 (size=93) 2024-11-27T12:28:47,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741939_1119 (size=93) 2024-11-27T12:28:47,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741939_1119 (size=93) 2024-11-27T12:28:47,261 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T12:28:47,261 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732710527087) 2024-11-27T12:28:47,275 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=449 (was 443) Potentially hanging thread: IPC Client (1612229954) connection to localhost/127.0.0.1:38485 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_705662131_22 at /127.0.0.1:33252 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_705662131_22 at /127.0.0.1:53788 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:38485 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_705662131_22 at /127.0.0.1:50358 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=202 (was 220), ProcessCount=11 (was 11), AvailableMemoryMB=8724 (was 8755) 2024-11-27T12:28:47,275 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-27T12:28:47,275 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T12:28:47,276 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T12:28:47,276 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:28:47,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:28:47,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:28:47,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:28:47,276 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T12:28:47,277 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=805536183, stopped=false 2024-11-27T12:28:47,277 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1ba5af43fd99,46311,1732710488723 2024-11-27T12:28:47,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T12:28:47,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T12:28:47,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T12:28:47,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:47,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:47,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:47,314 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T12:28:47,315 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:28:47,315 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T12:28:47,316 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:28:47,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:28:47,316 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:28:47,316 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:28:47,316 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1ba5af43fd99,41787,1732710489587' ***** 2024-11-27T12:28:47,316 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T12:28:47,316 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1ba5af43fd99,44849,1732710489640' ***** 2024-11-27T12:28:47,316 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T12:28:47,316 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T12:28:47,317 INFO [RS:1;1ba5af43fd99:41787 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T12:28:47,317 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T12:28:47,317 INFO [RS:1;1ba5af43fd99:41787 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T12:28:47,317 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.HRegionServer(3091): Received CLOSE for 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:47,317 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T12:28:47,317 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T12:28:47,317 INFO [RS:2;1ba5af43fd99:44849 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T12:28:47,317 INFO [RS:2;1ba5af43fd99:44849 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T12:28:47,317 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.HRegionServer(959): stopping server 1ba5af43fd99,41787,1732710489587 2024-11-27T12:28:47,317 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.HRegionServer(959): stopping server 1ba5af43fd99,44849,1732710489640 2024-11-27T12:28:47,317 INFO [RS:1;1ba5af43fd99:41787 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T12:28:47,317 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5556f2e573f63651ecac0666fe4456cb, disabling compactions & flushes 2024-11-27T12:28:47,317 INFO [RS:2;1ba5af43fd99:44849 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T12:28:47,317 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:47,317 INFO [RS:1;1ba5af43fd99:41787 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;1ba5af43fd99:41787. 2024-11-27T12:28:47,317 INFO [RS:2;1ba5af43fd99:44849 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;1ba5af43fd99:44849. 2024-11-27T12:28:47,318 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:47,318 DEBUG [RS:2;1ba5af43fd99:44849 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:28:47,318 DEBUG [RS:1;1ba5af43fd99:41787 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:28:47,318 DEBUG [RS:2;1ba5af43fd99:44849 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:28:47,318 DEBUG [RS:1;1ba5af43fd99:41787 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:28:47,318 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T12:28:47,318 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T12:28:47,318 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T12:28:47,318 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-27T12:28:47,318 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T12:28:47,318 DEBUG [RS:1;1ba5af43fd99:41787 {}] regionserver.HRegionServer(1325): Online Regions={5556f2e573f63651ecac0666fe4456cb=testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb.} 2024-11-27T12:28:47,318 DEBUG [RS:1;1ba5af43fd99:41787 {}] regionserver.HRegionServer(1351): Waiting on 5556f2e573f63651ecac0666fe4456cb 2024-11-27T12:28:47,318 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. after waiting 0 ms 2024-11-27T12:28:47,318 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-27T12:28:47,318 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:47,318 DEBUG [RS:2;1ba5af43fd99:44849 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-27T12:28:47,319 DEBUG [RS:2;1ba5af43fd99:44849 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T12:28:47,319 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T12:28:47,319 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T12:28:47,319 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T12:28:47,319 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T12:28:47,319 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T12:28:47,319 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-27T12:28:47,324 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5556f2e573f63651ecac0666fe4456cb/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-27T12:28:47,325 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:47,325 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5556f2e573f63651ecac0666fe4456cb: Waiting for close lock at 1732710527317Running coprocessor pre-close hooks at 1732710527317Disabling compacts and flushes for region at 1732710527317Disabling writes for close at 1732710527318 (+1 ms)Writing region close event to WAL at 1732710527320 (+2 ms)Running coprocessor post-close hooks at 1732710527325 (+5 ms)Closed at 1732710527325 2024-11-27T12:28:47,325 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb. 2024-11-27T12:28:47,325 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/.tmp/info/c5aa6f5ceb664e9ba2f962c58fd6218b is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732710503633.5556f2e573f63651ecac0666fe4456cb./info:regioninfo/1732710507638/Put/seqid=0 2024-11-27T12:28:47,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741943_1123 (size=6778) 2024-11-27T12:28:47,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741943_1123 (size=6778) 2024-11-27T12:28:47,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741943_1123 (size=6778) 2024-11-27T12:28:47,334 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/.tmp/info/c5aa6f5ceb664e9ba2f962c58fd6218b 2024-11-27T12:28:47,339 INFO [regionserver/1ba5af43fd99:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T12:28:47,339 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/.tmp/info/c5aa6f5ceb664e9ba2f962c58fd6218b as hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/info/c5aa6f5ceb664e9ba2f962c58fd6218b 2024-11-27T12:28:47,344 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/info/c5aa6f5ceb664e9ba2f962c58fd6218b, entries=8, sequenceid=23, filesize=6.6 K 2024-11-27T12:28:47,345 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 26ms, sequenceid=23, compaction requested=false 2024-11-27T12:28:47,345 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T12:28:47,350 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-27T12:28:47,351 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T12:28:47,351 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T12:28:47,351 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732710527319Running coprocessor pre-close hooks at 1732710527319Disabling compacts and flushes for region at 1732710527319Disabling writes for close at 1732710527319Obtaining lock to block concurrent updates at 1732710527319Preparing flush snapshotting stores in 1588230740 at 1732710527319Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1732710527320 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732710527321 (+1 ms)Flushing 1588230740/info: creating writer at 1732710527321Flushing 1588230740/info: appending metadata at 1732710527325 (+4 ms)Flushing 1588230740/info: closing flushed file at 1732710527325Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3aca8533: reopening flushed file at 1732710527339 (+14 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 26ms, sequenceid=23, compaction requested=false at 1732710527345 (+6 ms)Writing region close event to WAL at 1732710527347 (+2 ms)Running coprocessor post-close hooks at 1732710527351 (+4 ms)Closed at 1732710527351 2024-11-27T12:28:47,351 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T12:28:47,369 INFO [regionserver/1ba5af43fd99:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T12:28:47,518 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.HRegionServer(976): stopping server 1ba5af43fd99,41787,1732710489587; all regions closed. 2024-11-27T12:28:47,519 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.HRegionServer(976): stopping server 1ba5af43fd99,44849,1732710489640; all regions closed. 2024-11-27T12:28:47,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741835_1011 (size=3433) 2024-11-27T12:28:47,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741835_1011 (size=3433) 2024-11-27T12:28:47,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741835_1011 (size=3433) 2024-11-27T12:28:47,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741893_1071 (size=2422) 2024-11-27T12:28:47,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741893_1071 (size=2422) 2024-11-27T12:28:47,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741893_1071 (size=2422) 2024-11-27T12:28:47,523 DEBUG [RS:1;1ba5af43fd99:41787 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/oldWALs 2024-11-27T12:28:47,523 INFO [RS:1;1ba5af43fd99:41787 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1ba5af43fd99%2C41787%2C1732710489587:(num 1732710491169) 2024-11-27T12:28:47,523 DEBUG [RS:1;1ba5af43fd99:41787 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:28:47,523 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T12:28:47,524 INFO [RS:1;1ba5af43fd99:41787 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T12:28:47,524 DEBUG [RS:2;1ba5af43fd99:44849 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/oldWALs 2024-11-27T12:28:47,524 INFO [RS:1;1ba5af43fd99:41787 {}] hbase.ChoreService(370): Chore service for: regionserver/1ba5af43fd99:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T12:28:47,524 INFO [RS:2;1ba5af43fd99:44849 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1ba5af43fd99%2C44849%2C1732710489640.meta:.meta(num 1732710506795) 2024-11-27T12:28:47,524 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T12:28:47,524 INFO [regionserver/1ba5af43fd99:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T12:28:47,524 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T12:28:47,524 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T12:28:47,524 INFO [RS:1;1ba5af43fd99:41787 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T12:28:47,524 INFO [RS:1;1ba5af43fd99:41787 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41787 2024-11-27T12:28:47,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741833_1009 (size=93) 2024-11-27T12:28:47,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741833_1009 (size=93) 2024-11-27T12:28:47,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741833_1009 (size=93) 2024-11-27T12:28:47,528 DEBUG [RS:2;1ba5af43fd99:44849 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/oldWALs 2024-11-27T12:28:47,528 INFO [RS:2;1ba5af43fd99:44849 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1ba5af43fd99%2C44849%2C1732710489640:(num 1732710491115) 2024-11-27T12:28:47,528 DEBUG [RS:2;1ba5af43fd99:44849 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:28:47,529 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T12:28:47,529 INFO [RS:2;1ba5af43fd99:44849 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T12:28:47,529 INFO [RS:2;1ba5af43fd99:44849 {}] hbase.ChoreService(370): Chore service for: regionserver/1ba5af43fd99:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T12:28:47,529 INFO [RS:2;1ba5af43fd99:44849 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T12:28:47,529 INFO [regionserver/1ba5af43fd99:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T12:28:47,529 INFO [RS:2;1ba5af43fd99:44849 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44849 2024-11-27T12:28:47,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T12:28:47,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1ba5af43fd99,41787,1732710489587 2024-11-27T12:28:47,535 INFO [RS:1;1ba5af43fd99:41787 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T12:28:47,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1ba5af43fd99,44849,1732710489640 2024-11-27T12:28:47,546 INFO [RS:2;1ba5af43fd99:44849 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T12:28:47,556 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1ba5af43fd99,41787,1732710489587] 2024-11-27T12:28:47,577 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1ba5af43fd99,41787,1732710489587 already deleted, retry=false 2024-11-27T12:28:47,577 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1ba5af43fd99,41787,1732710489587 expired; onlineServers=1 2024-11-27T12:28:47,578 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1ba5af43fd99,44849,1732710489640] 2024-11-27T12:28:47,588 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1ba5af43fd99,44849,1732710489640 already deleted, retry=false 2024-11-27T12:28:47,588 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1ba5af43fd99,44849,1732710489640 expired; onlineServers=0 2024-11-27T12:28:47,588 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1ba5af43fd99,46311,1732710488723' ***** 2024-11-27T12:28:47,588 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T12:28:47,588 INFO [M:0;1ba5af43fd99:46311 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T12:28:47,588 INFO [M:0;1ba5af43fd99:46311 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T12:28:47,588 DEBUG [M:0;1ba5af43fd99:46311 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T12:28:47,588 DEBUG [M:0;1ba5af43fd99:46311 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T12:28:47,588 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T12:28:47,588 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster-HFileCleaner.large.0-1732710490780 {}] cleaner.HFileCleaner(306): Exit Thread[master/1ba5af43fd99:0:becomeActiveMaster-HFileCleaner.large.0-1732710490780,5,FailOnTimeoutGroup] 2024-11-27T12:28:47,588 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster-HFileCleaner.small.0-1732710490784 {}] cleaner.HFileCleaner(306): Exit Thread[master/1ba5af43fd99:0:becomeActiveMaster-HFileCleaner.small.0-1732710490784,5,FailOnTimeoutGroup] 2024-11-27T12:28:47,589 INFO [M:0;1ba5af43fd99:46311 {}] hbase.ChoreService(370): Chore service for: master/1ba5af43fd99:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T12:28:47,589 INFO [M:0;1ba5af43fd99:46311 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T12:28:47,589 DEBUG [M:0;1ba5af43fd99:46311 {}] master.HMaster(1795): Stopping service threads 2024-11-27T12:28:47,589 INFO [M:0;1ba5af43fd99:46311 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T12:28:47,589 INFO [M:0;1ba5af43fd99:46311 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T12:28:47,590 INFO [M:0;1ba5af43fd99:46311 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T12:28:47,590 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T12:28:47,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T12:28:47,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:28:47,599 DEBUG [M:0;1ba5af43fd99:46311 {}] zookeeper.ZKUtil(347): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-27T12:28:47,599 WARN [M:0;1ba5af43fd99:46311 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T12:28:47,599 INFO [M:0;1ba5af43fd99:46311 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/.lastflushedseqids 2024-11-27T12:28:47,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741944_1124 (size=119) 2024-11-27T12:28:47,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741944_1124 (size=119) 2024-11-27T12:28:47,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741944_1124 (size=119) 2024-11-27T12:28:47,615 INFO [M:0;1ba5af43fd99:46311 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T12:28:47,615 INFO [M:0;1ba5af43fd99:46311 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T12:28:47,615 DEBUG [M:0;1ba5af43fd99:46311 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T12:28:47,615 INFO [M:0;1ba5af43fd99:46311 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:28:47,615 DEBUG [M:0;1ba5af43fd99:46311 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:28:47,615 DEBUG [M:0;1ba5af43fd99:46311 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T12:28:47,616 DEBUG [M:0;1ba5af43fd99:46311 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:28:47,616 INFO [M:0;1ba5af43fd99:46311 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.40 KB heapSize=102.70 KB 2024-11-27T12:28:47,633 DEBUG [M:0;1ba5af43fd99:46311 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/319f5722dc7346ea98fb84422febfd34 is 82, key is hbase:meta,,1/info:regioninfo/1732710506959/Put/seqid=0 2024-11-27T12:28:47,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741945_1125 (size=6063) 2024-11-27T12:28:47,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741945_1125 (size=6063) 2024-11-27T12:28:47,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741945_1125 (size=6063) 2024-11-27T12:28:47,640 INFO [M:0;1ba5af43fd99:46311 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/319f5722dc7346ea98fb84422febfd34 2024-11-27T12:28:47,656 INFO [RS:1;1ba5af43fd99:41787 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T12:28:47,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:28:47,657 INFO [RS:1;1ba5af43fd99:41787 {}] regionserver.HRegionServer(1031): Exiting; stopping=1ba5af43fd99,41787,1732710489587; zookeeper connection closed. 2024-11-27T12:28:47,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41787-0x1017c43d3070002, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:28:47,657 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@351eac6a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@351eac6a 2024-11-27T12:28:47,658 DEBUG [M:0;1ba5af43fd99:46311 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dbc565883c3640d0b2fe34afe67c9beb is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732710504094/Put/seqid=0 2024-11-27T12:28:47,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741946_1126 (size=7907) 2024-11-27T12:28:47,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741946_1126 (size=7907) 2024-11-27T12:28:47,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741946_1126 (size=7907) 2024-11-27T12:28:47,666 INFO [M:0;1ba5af43fd99:46311 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.17 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dbc565883c3640d0b2fe34afe67c9beb 2024-11-27T12:28:47,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:28:47,667 INFO [RS:2;1ba5af43fd99:44849 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T12:28:47,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44849-0x1017c43d3070003, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:28:47,667 INFO [RS:2;1ba5af43fd99:44849 {}] regionserver.HRegionServer(1031): Exiting; stopping=1ba5af43fd99,44849,1732710489640; zookeeper connection closed. 2024-11-27T12:28:47,667 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2ffa52 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2ffa52 2024-11-27T12:28:47,668 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-27T12:28:47,670 INFO [M:0;1ba5af43fd99:46311 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dbc565883c3640d0b2fe34afe67c9beb 2024-11-27T12:28:47,684 DEBUG [M:0;1ba5af43fd99:46311 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7e365a5ceff246fba8c832321701a331 is 69, key is 1ba5af43fd99,41787,1732710489587/rs:state/1732710490936/Put/seqid=0 2024-11-27T12:28:47,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741947_1127 (size=5440) 2024-11-27T12:28:47,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741947_1127 (size=5440) 2024-11-27T12:28:47,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741947_1127 (size=5440) 2024-11-27T12:28:47,690 INFO [M:0;1ba5af43fd99:46311 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7e365a5ceff246fba8c832321701a331 2024-11-27T12:28:47,695 INFO [M:0;1ba5af43fd99:46311 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7e365a5ceff246fba8c832321701a331 2024-11-27T12:28:47,697 DEBUG [M:0;1ba5af43fd99:46311 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/319f5722dc7346ea98fb84422febfd34 as hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/319f5722dc7346ea98fb84422febfd34 2024-11-27T12:28:47,702 INFO [M:0;1ba5af43fd99:46311 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/319f5722dc7346ea98fb84422febfd34, entries=14, sequenceid=207, filesize=5.9 K 2024-11-27T12:28:47,702 DEBUG [M:0;1ba5af43fd99:46311 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dbc565883c3640d0b2fe34afe67c9beb as hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dbc565883c3640d0b2fe34afe67c9beb 2024-11-27T12:28:47,707 INFO [M:0;1ba5af43fd99:46311 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dbc565883c3640d0b2fe34afe67c9beb 2024-11-27T12:28:47,707 INFO [M:0;1ba5af43fd99:46311 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dbc565883c3640d0b2fe34afe67c9beb, entries=21, sequenceid=207, filesize=7.7 K 2024-11-27T12:28:47,708 DEBUG [M:0;1ba5af43fd99:46311 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7e365a5ceff246fba8c832321701a331 as hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7e365a5ceff246fba8c832321701a331 2024-11-27T12:28:47,712 INFO [M:0;1ba5af43fd99:46311 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7e365a5ceff246fba8c832321701a331 2024-11-27T12:28:47,712 INFO [M:0;1ba5af43fd99:46311 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38485/user/jenkins/test-data/db7a3f94-68d6-9f87-ef63-c2f93aa1a606/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7e365a5ceff246fba8c832321701a331, entries=3, sequenceid=207, filesize=5.3 K 2024-11-27T12:28:47,713 INFO [M:0;1ba5af43fd99:46311 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 97ms, sequenceid=207, compaction requested=false 2024-11-27T12:28:47,715 INFO [M:0;1ba5af43fd99:46311 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:28:47,715 DEBUG [M:0;1ba5af43fd99:46311 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732710527615Disabling compacts and flushes for region at 1732710527615Disabling writes for close at 1732710527615Obtaining lock to block concurrent updates at 1732710527616 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732710527616Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85398, getHeapSize=105104, getOffHeapSize=0, getCellsCount=248 at 1732710527616Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732710527617 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732710527617Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732710527633 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732710527633Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732710527644 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732710527657 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732710527657Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732710527670 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732710527683 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732710527683Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@294b827c: reopening flushed file at 1732710527695 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@776502e1: reopening flushed file at 1732710527702 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6137d1fe: reopening flushed file at 1732710527707 (+5 ms)Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 97ms, sequenceid=207, compaction requested=false at 1732710527713 (+6 ms)Writing region close event to WAL at 1732710527715 (+2 ms)Closed at 1732710527715 2024-11-27T12:28:47,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38877 is added to blk_1073741830_1006 (size=99436) 2024-11-27T12:28:47,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37469 is added to blk_1073741830_1006 (size=99436) 2024-11-27T12:28:47,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38365 is added to blk_1073741830_1006 (size=99436) 2024-11-27T12:28:47,719 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T12:28:47,719 INFO [M:0;1ba5af43fd99:46311 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T12:28:47,719 INFO [M:0;1ba5af43fd99:46311 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46311 2024-11-27T12:28:47,719 INFO [M:0;1ba5af43fd99:46311 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T12:28:47,825 INFO [M:0;1ba5af43fd99:46311 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T12:28:47,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:28:47,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46311-0x1017c43d3070000, quorum=127.0.0.1:58880, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:28:47,833 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255/wal.1732710517496 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255/wal.1732710517496 (inode 16688) Holder DFSClient_NONMAPREDUCE_-862841147_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732710508255/wal.1732710517496 (inode 16688) Holder DFSClient_NONMAPREDUCE_-862841147_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-27T12:28:47,837 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707/wal.1732710508107 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707/wal.1732710508107 (inode 16665) Holder DFSClient_NONMAPREDUCE_-862841147_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732710507707/wal.1732710508107 (inode 16665) Holder DFSClient_NONMAPREDUCE_-862841147_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-27T12:28:47,838 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732710499217/wal.1732710499341 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:47,840 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679/wal.1732710518106 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679/wal.1732710518106 (inode 16714) Holder DFSClient_NONMAPREDUCE_-862841147_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732710517679/wal.1732710518106 (inode 16714) Holder DFSClient_NONMAPREDUCE_-862841147_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-27T12:28:47,840 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732710518251/wal.1732710518308 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:47,840 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732710522693/wal.1732710522876 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:47,840 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732710492877/wal.1732710492942 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:28:47,842 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal.1732710492742 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal.1732710492742 (inode 16485) Holder DFSClient_NONMAPREDUCE_-862841147_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732710492461/wal.1732710492742 (inode 16485) Holder DFSClient_NONMAPREDUCE_-862841147_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-27T12:28:47,844 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal.1732710492235 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal.1732710492235 (inode 16462) Holder DFSClient_NONMAPREDUCE_-862841147_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732710491985/wal.1732710492235 (inode 16462) Holder DFSClient_NONMAPREDUCE_-862841147_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-27T12:28:47,849 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@638bb008{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T12:28:47,851 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@394c588e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T12:28:47,851 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T12:28:47,851 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3cda6c9e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T12:28:47,851 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4870c3fc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/hadoop.log.dir/,STOPPED} 2024-11-27T12:28:47,853 WARN [BP-39017082-172.17.0.3-1732710484128 heartbeating to localhost/127.0.0.1:38485 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T12:28:47,853 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T12:28:47,853 WARN [BP-39017082-172.17.0.3-1732710484128 heartbeating to localhost/127.0.0.1:38485 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-39017082-172.17.0.3-1732710484128 (Datanode Uuid d8a095d7-047e-41bc-87f0-5bd820a5ee93) service to localhost/127.0.0.1:38485 2024-11-27T12:28:47,853 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T12:28:47,855 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/data/data5/current/BP-39017082-172.17.0.3-1732710484128 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T12:28:47,855 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/data/data6/current/BP-39017082-172.17.0.3-1732710484128 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T12:28:47,855 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T12:28:47,857 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@bdffcb9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T12:28:47,857 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@55b08a17{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T12:28:47,857 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T12:28:47,857 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2ea2c5fa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T12:28:47,857 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5c155ec7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/hadoop.log.dir/,STOPPED} 2024-11-27T12:28:47,858 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T12:28:47,858 WARN [BP-39017082-172.17.0.3-1732710484128 heartbeating to localhost/127.0.0.1:38485 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T12:28:47,858 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T12:28:47,859 WARN [BP-39017082-172.17.0.3-1732710484128 heartbeating to localhost/127.0.0.1:38485 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-39017082-172.17.0.3-1732710484128 (Datanode Uuid c049323e-5ead-432d-9a1c-c0835999441a) service to localhost/127.0.0.1:38485 2024-11-27T12:28:47,859 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/data/data3/current/BP-39017082-172.17.0.3-1732710484128 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T12:28:47,859 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/data/data4/current/BP-39017082-172.17.0.3-1732710484128 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T12:28:47,860 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T12:28:47,865 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@53a60ac5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T12:28:47,865 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b9d2b29{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T12:28:47,865 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T12:28:47,866 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f0d3e1c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T12:28:47,866 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@15fab457{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/hadoop.log.dir/,STOPPED} 2024-11-27T12:28:47,867 WARN [BP-39017082-172.17.0.3-1732710484128 heartbeating to localhost/127.0.0.1:38485 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T12:28:47,867 WARN [BP-39017082-172.17.0.3-1732710484128 heartbeating to localhost/127.0.0.1:38485 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-39017082-172.17.0.3-1732710484128 (Datanode Uuid 1859aba2-a42a-4521-b053-d968c4078117) service to localhost/127.0.0.1:38485 2024-11-27T12:28:47,868 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T12:28:47,868 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T12:28:47,868 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/data/data1/current/BP-39017082-172.17.0.3-1732710484128 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T12:28:47,868 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/cluster_24f0edf1-cfa9-397a-d877-774d2b866401/data/data2/current/BP-39017082-172.17.0.3-1732710484128 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T12:28:47,869 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T12:28:47,893 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3c411b81{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T12:28:47,894 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@71f91499{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T12:28:47,894 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T12:28:47,894 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a9265d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T12:28:47,894 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3c1cc886{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6e44a988-cb0e-a124-72db-26dbf5a42db5/hadoop.log.dir/,STOPPED} 2024-11-27T12:28:47,904 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T12:28:47,959 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down