2024-11-23 22:49:04,421 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24569dba 2024-11-23 22:49:04,438 main DEBUG Took 0.014566 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-23 22:49:04,439 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-23 22:49:04,439 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-23 22:49:04,441 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-23 22:49:04,442 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:49:04,476 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-23 22:49:04,493 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:49:04,494 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:49:04,495 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:49:04,496 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:49:04,496 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:49:04,497 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:49:04,498 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:49:04,498 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:49:04,499 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:49:04,499 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:49:04,501 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:49:04,501 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:49:04,502 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:49:04,502 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:49:04,503 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:49:04,503 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:49:04,504 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:49:04,504 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:49:04,505 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:49:04,505 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:49:04,506 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:49:04,506 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:49:04,507 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:49:04,508 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:49:04,508 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:49:04,509 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-23 22:49:04,511 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:49:04,512 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-23 22:49:04,515 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-23 22:49:04,516 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-23 22:49:04,517 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-23 22:49:04,518 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-23 22:49:04,529 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-23 22:49:04,532 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-23 22:49:04,535 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-23 22:49:04,535 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-23 22:49:04,536 main DEBUG createAppenders(={Console}) 2024-11-23 22:49:04,537 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24569dba initialized 2024-11-23 22:49:04,537 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24569dba 2024-11-23 22:49:04,537 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24569dba OK. 2024-11-23 22:49:04,538 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-23 22:49:04,538 main DEBUG OutputStream closed 2024-11-23 22:49:04,539 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-23 22:49:04,539 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-23 22:49:04,539 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@49c7b90e OK 2024-11-23 22:49:04,649 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-23 22:49:04,653 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-23 22:49:04,655 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-23 22:49:04,656 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-23 22:49:04,657 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-23 22:49:04,658 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-23 22:49:04,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-23 22:49:04,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-23 22:49:04,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-23 22:49:04,660 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-23 22:49:04,660 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-23 22:49:04,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-23 22:49:04,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-23 22:49:04,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-23 22:49:04,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-23 22:49:04,664 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-23 22:49:04,664 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-23 22:49:04,665 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-23 22:49:04,668 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23 22:49:04,669 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@35432107) with optional ClassLoader: null 2024-11-23 22:49:04,669 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-23 22:49:04,670 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@35432107] started OK. 2024-11-23T22:49:04,689 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestHBaseWalOnEC timeout: 26 mins 2024-11-23 22:49:04,692 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-23 22:49:04,693 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23T22:49:05,125 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6 2024-11-23T22:49:05,169 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/cluster_10dec550-ad07-9099-3875-13a2f1d6fb1f, deleteOnExit=true 2024-11-23T22:49:05,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/test.cache.data in system properties and HBase conf 2024-11-23T22:49:05,172 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/hadoop.tmp.dir in system properties and HBase conf 2024-11-23T22:49:05,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/hadoop.log.dir in system properties and HBase conf 2024-11-23T22:49:05,174 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-23T22:49:05,175 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-23T22:49:05,176 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-23T22:49:05,293 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-23T22:49:05,430 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:49:05,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-23T22:49:05,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-23T22:49:05,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-23T22:49:05,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T22:49:05,443 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-23T22:49:05,444 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-23T22:49:05,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T22:49:05,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T22:49:05,446 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-23T22:49:05,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/nfs.dump.dir in system properties and HBase conf 2024-11-23T22:49:05,448 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/java.io.tmpdir in system properties and HBase conf 2024-11-23T22:49:05,448 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T22:49:05,449 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-23T22:49:05,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-23T22:49:06,805 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-23T22:49:06,912 INFO [Time-limited test {}] log.Log(170): Logging initialized @3392ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-23T22:49:07,015 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T22:49:07,117 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T22:49:07,174 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T22:49:07,174 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T22:49:07,178 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T22:49:07,211 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T22:49:07,219 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@760c69c0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/hadoop.log.dir/,AVAILABLE} 2024-11-23T22:49:07,221 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ce709a8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T22:49:07,514 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@62d6efd9{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/java.io.tmpdir/jetty-localhost-41903-hadoop-hdfs-3_4_1-tests_jar-_-any-15501415082662278460/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T22:49:07,524 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d35a1{HTTP/1.1, (http/1.1)}{localhost:41903} 2024-11-23T22:49:07,525 INFO [Time-limited test {}] server.Server(415): Started @4006ms 2024-11-23T22:49:08,218 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T22:49:08,229 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T22:49:08,232 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T22:49:08,232 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T22:49:08,233 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T22:49:08,236 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3665148e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/hadoop.log.dir/,AVAILABLE} 2024-11-23T22:49:08,237 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3b3a0659{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T22:49:08,352 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5b23cf15{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/java.io.tmpdir/jetty-localhost-34551-hadoop-hdfs-3_4_1-tests_jar-_-any-7584506988424146735/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T22:49:08,353 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7f9e5902{HTTP/1.1, (http/1.1)}{localhost:34551} 2024-11-23T22:49:08,353 INFO [Time-limited test {}] server.Server(415): Started @4835ms 2024-11-23T22:49:08,414 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T22:49:08,633 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T22:49:08,649 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T22:49:08,685 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T22:49:08,685 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T22:49:08,686 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T22:49:08,689 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5435fd88{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/hadoop.log.dir/,AVAILABLE} 2024-11-23T22:49:08,690 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@65cd6e19{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T22:49:08,838 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@14402056{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/java.io.tmpdir/jetty-localhost-42877-hadoop-hdfs-3_4_1-tests_jar-_-any-18401743469167617709/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T22:49:08,839 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3e1cb3ec{HTTP/1.1, (http/1.1)}{localhost:42877} 2024-11-23T22:49:08,839 INFO [Time-limited test {}] server.Server(415): Started @5321ms 2024-11-23T22:49:08,841 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T22:49:08,905 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T22:49:08,911 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T22:49:08,915 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T22:49:08,915 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T22:49:08,916 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T22:49:08,917 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@233bb3ea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/hadoop.log.dir/,AVAILABLE} 2024-11-23T22:49:08,917 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@146c020c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T22:49:09,037 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2a9ecb50{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/java.io.tmpdir/jetty-localhost-40223-hadoop-hdfs-3_4_1-tests_jar-_-any-13962164949005567609/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T22:49:09,038 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4067fd8a{HTTP/1.1, (http/1.1)}{localhost:40223} 2024-11-23T22:49:09,038 INFO [Time-limited test {}] server.Server(415): Started @5520ms 2024-11-23T22:49:09,041 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T22:49:10,235 WARN [Thread-120 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/cluster_10dec550-ad07-9099-3875-13a2f1d6fb1f/data/data1/current/BP-868742450-172.17.0.2-1732402146260/current, will proceed with Du for space computation calculation, 2024-11-23T22:49:10,235 WARN [Thread-121 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/cluster_10dec550-ad07-9099-3875-13a2f1d6fb1f/data/data2/current/BP-868742450-172.17.0.2-1732402146260/current, will proceed with Du for space computation calculation, 2024-11-23T22:49:10,261 WARN [Thread-126 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/cluster_10dec550-ad07-9099-3875-13a2f1d6fb1f/data/data3/current/BP-868742450-172.17.0.2-1732402146260/current, will proceed with Du for space computation calculation, 2024-11-23T22:49:10,267 WARN [Thread-127 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/cluster_10dec550-ad07-9099-3875-13a2f1d6fb1f/data/data4/current/BP-868742450-172.17.0.2-1732402146260/current, will proceed with Du for space computation calculation, 2024-11-23T22:49:10,312 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T22:49:10,317 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T22:49:10,372 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x25ff991cda38d36 with lease ID 0x9a2e32bb6b24dc56: Processing first storage report for DS-5999b11b-fe98-468f-bf4c-ea395a07b98b from datanode DatanodeRegistration(127.0.0.1:43763, datanodeUuid=ff009774-bb68-4f04-a785-f72b51c43e68, infoPort=42995, infoSecurePort=0, ipcPort=38799, storageInfo=lv=-57;cid=testClusterID;nsid=1079746219;c=1732402146260) 2024-11-23T22:49:10,373 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x25ff991cda38d36 with lease ID 0x9a2e32bb6b24dc56: from storage DS-5999b11b-fe98-468f-bf4c-ea395a07b98b node DatanodeRegistration(127.0.0.1:43763, datanodeUuid=ff009774-bb68-4f04-a785-f72b51c43e68, infoPort=42995, infoSecurePort=0, ipcPort=38799, storageInfo=lv=-57;cid=testClusterID;nsid=1079746219;c=1732402146260), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-23T22:49:10,374 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xed93c525f960a444 with lease ID 0x9a2e32bb6b24dc55: Processing first storage report for DS-703608bf-71fd-4811-aa6f-e7553fea4492 from datanode DatanodeRegistration(127.0.0.1:36219, datanodeUuid=6222555d-d845-4165-87b4-a4b99c8e8037, infoPort=36559, infoSecurePort=0, ipcPort=33835, storageInfo=lv=-57;cid=testClusterID;nsid=1079746219;c=1732402146260) 2024-11-23T22:49:10,374 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xed93c525f960a444 with lease ID 0x9a2e32bb6b24dc55: from storage DS-703608bf-71fd-4811-aa6f-e7553fea4492 node DatanodeRegistration(127.0.0.1:36219, datanodeUuid=6222555d-d845-4165-87b4-a4b99c8e8037, infoPort=36559, infoSecurePort=0, ipcPort=33835, storageInfo=lv=-57;cid=testClusterID;nsid=1079746219;c=1732402146260), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T22:49:10,374 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x25ff991cda38d36 with lease ID 0x9a2e32bb6b24dc56: Processing first storage report for DS-c56c9149-f9ff-47a2-8844-6fa45090cf31 from datanode DatanodeRegistration(127.0.0.1:43763, datanodeUuid=ff009774-bb68-4f04-a785-f72b51c43e68, infoPort=42995, infoSecurePort=0, ipcPort=38799, storageInfo=lv=-57;cid=testClusterID;nsid=1079746219;c=1732402146260) 2024-11-23T22:49:10,374 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x25ff991cda38d36 with lease ID 0x9a2e32bb6b24dc56: from storage DS-c56c9149-f9ff-47a2-8844-6fa45090cf31 node DatanodeRegistration(127.0.0.1:43763, datanodeUuid=ff009774-bb68-4f04-a785-f72b51c43e68, infoPort=42995, infoSecurePort=0, ipcPort=38799, storageInfo=lv=-57;cid=testClusterID;nsid=1079746219;c=1732402146260), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T22:49:10,375 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xed93c525f960a444 with lease ID 0x9a2e32bb6b24dc55: Processing first storage report for DS-119d5619-7fe5-45bc-998f-93e73e8d353c from datanode DatanodeRegistration(127.0.0.1:36219, datanodeUuid=6222555d-d845-4165-87b4-a4b99c8e8037, infoPort=36559, infoSecurePort=0, ipcPort=33835, storageInfo=lv=-57;cid=testClusterID;nsid=1079746219;c=1732402146260) 2024-11-23T22:49:10,375 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xed93c525f960a444 with lease ID 0x9a2e32bb6b24dc55: from storage DS-119d5619-7fe5-45bc-998f-93e73e8d353c node DatanodeRegistration(127.0.0.1:36219, datanodeUuid=6222555d-d845-4165-87b4-a4b99c8e8037, infoPort=36559, infoSecurePort=0, ipcPort=33835, storageInfo=lv=-57;cid=testClusterID;nsid=1079746219;c=1732402146260), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-23T22:49:10,381 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/cluster_10dec550-ad07-9099-3875-13a2f1d6fb1f/data/data5/current/BP-868742450-172.17.0.2-1732402146260/current, will proceed with Du for space computation calculation, 2024-11-23T22:49:10,385 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/cluster_10dec550-ad07-9099-3875-13a2f1d6fb1f/data/data6/current/BP-868742450-172.17.0.2-1732402146260/current, will proceed with Du for space computation calculation, 2024-11-23T22:49:10,426 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T22:49:10,432 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb3eb0e86e9d39873 with lease ID 0x9a2e32bb6b24dc57: Processing first storage report for DS-c9e3d8d2-4dea-4625-b764-06ae2b7c1391 from datanode DatanodeRegistration(127.0.0.1:35543, datanodeUuid=abef8c5b-3c1a-4737-8cde-1b8f158c6a26, infoPort=43121, infoSecurePort=0, ipcPort=36205, storageInfo=lv=-57;cid=testClusterID;nsid=1079746219;c=1732402146260) 2024-11-23T22:49:10,432 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb3eb0e86e9d39873 with lease ID 0x9a2e32bb6b24dc57: from storage DS-c9e3d8d2-4dea-4625-b764-06ae2b7c1391 node DatanodeRegistration(127.0.0.1:35543, datanodeUuid=abef8c5b-3c1a-4737-8cde-1b8f158c6a26, infoPort=43121, infoSecurePort=0, ipcPort=36205, storageInfo=lv=-57;cid=testClusterID;nsid=1079746219;c=1732402146260), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-23T22:49:10,433 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb3eb0e86e9d39873 with lease ID 0x9a2e32bb6b24dc57: Processing first storage report for DS-e73a79b5-af0f-45ec-bfc3-06885e50b11d from datanode DatanodeRegistration(127.0.0.1:35543, datanodeUuid=abef8c5b-3c1a-4737-8cde-1b8f158c6a26, infoPort=43121, infoSecurePort=0, ipcPort=36205, storageInfo=lv=-57;cid=testClusterID;nsid=1079746219;c=1732402146260) 2024-11-23T22:49:10,433 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb3eb0e86e9d39873 with lease ID 0x9a2e32bb6b24dc57: from storage DS-e73a79b5-af0f-45ec-bfc3-06885e50b11d node DatanodeRegistration(127.0.0.1:35543, datanodeUuid=abef8c5b-3c1a-4737-8cde-1b8f158c6a26, infoPort=43121, infoSecurePort=0, ipcPort=36205, storageInfo=lv=-57;cid=testClusterID;nsid=1079746219;c=1732402146260), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T22:49:10,514 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6 2024-11-23T22:49:10,620 WARN [Time-limited test {}] erasurecode.ErasureCodeNative(55): ISA-L support is not available in your platform... using builtin-java codec where applicable 2024-11-23T22:49:10,714 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestHBaseWalOnEC#testReadWrite[0] Thread=157, OpenFileDescriptor=391, MaxFileDescriptor=1048576, SystemLoadAverage=587, ProcessCount=11, AvailableMemoryMB=7211 2024-11-23T22:49:10,717 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-23T22:49:10,727 INFO [Time-limited test {}] hbase.HBaseTestingUtil(821): NOT STARTING DFS 2024-11-23T22:49:10,873 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/cluster_10dec550-ad07-9099-3875-13a2f1d6fb1f/zookeeper_0, clientPort=57490, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/cluster_10dec550-ad07-9099-3875-13a2f1d6fb1f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/cluster_10dec550-ad07-9099-3875-13a2f1d6fb1f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T22:49:10,888 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57490 2024-11-23T22:49:10,934 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:10,940 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:11,068 WARN [Time-limited test {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:11,069 WARN [Time-limited test {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:11,145 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_801357093_22 at /127.0.0.1:52646 [Receiving block BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775792_1001] {}] datanode.DataXceiver(331): 127.0.0.1:36219:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52646 dst: /127.0.0.1:36219 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:11,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36219 is added to blk_-9223372036854775792_1002 (size=7) 2024-11-23T22:49:11,189 WARN [Time-limited test {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-23T22:49:11,201 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef with version=8 2024-11-23T22:49:11,201 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/hbase-staging 2024-11-23T22:49:11,325 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-23T22:49:11,625 INFO [Time-limited test {}] client.ConnectionUtils(128): master/dd96e8dc8369:0 server-side Connection retries=45 2024-11-23T22:49:11,637 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:11,637 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:11,643 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T22:49:11,643 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:11,644 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T22:49:11,827 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T22:49:11,904 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-23T22:49:11,917 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-23T22:49:11,922 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T22:49:11,955 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 53602 (auto-detected) 2024-11-23T22:49:11,957 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-23T22:49:11,979 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32885 2024-11-23T22:49:12,003 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:32885 connecting to ZooKeeper ensemble=127.0.0.1:57490 2024-11-23T22:49:12,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:328850x0, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T22:49:12,154 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:32885-0x10169e2f9690000 connected 2024-11-23T22:49:12,251 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:12,255 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:12,267 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:49:12,273 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef, hbase.cluster.distributed=false 2024-11-23T22:49:12,305 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T22:49:12,311 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32885 2024-11-23T22:49:12,312 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32885 2024-11-23T22:49:12,316 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32885 2024-11-23T22:49:12,317 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32885 2024-11-23T22:49:12,317 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32885 2024-11-23T22:49:12,433 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd96e8dc8369:0 server-side Connection retries=45 2024-11-23T22:49:12,435 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:12,435 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:12,436 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T22:49:12,436 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:12,436 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T22:49:12,439 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T22:49:12,441 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T22:49:12,442 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38153 2024-11-23T22:49:12,444 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38153 connecting to ZooKeeper ensemble=127.0.0.1:57490 2024-11-23T22:49:12,445 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:12,450 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:12,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:381530x0, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T22:49:12,481 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:381530x0, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:49:12,482 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38153-0x10169e2f9690001 connected 2024-11-23T22:49:12,486 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T22:49:12,493 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T22:49:12,496 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T22:49:12,502 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T22:49:12,502 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38153 2024-11-23T22:49:12,503 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38153 2024-11-23T22:49:12,503 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38153 2024-11-23T22:49:12,504 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38153 2024-11-23T22:49:12,507 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38153 2024-11-23T22:49:12,524 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd96e8dc8369:0 server-side Connection retries=45 2024-11-23T22:49:12,525 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:12,525 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:12,526 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T22:49:12,526 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:12,526 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T22:49:12,526 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T22:49:12,526 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T22:49:12,527 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35195 2024-11-23T22:49:12,529 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35195 connecting to ZooKeeper ensemble=127.0.0.1:57490 2024-11-23T22:49:12,530 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:12,533 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:12,544 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:351950x0, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T22:49:12,545 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:351950x0, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:49:12,546 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T22:49:12,548 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35195-0x10169e2f9690002 connected 2024-11-23T22:49:12,552 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T22:49:12,553 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T22:49:12,556 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T22:49:12,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35195 2024-11-23T22:49:12,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35195 2024-11-23T22:49:12,564 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35195 2024-11-23T22:49:12,568 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35195 2024-11-23T22:49:12,569 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35195 2024-11-23T22:49:12,587 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd96e8dc8369:0 server-side Connection retries=45 2024-11-23T22:49:12,587 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:12,588 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:12,588 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T22:49:12,588 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:12,588 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T22:49:12,588 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T22:49:12,589 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T22:49:12,590 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45809 2024-11-23T22:49:12,591 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45809 connecting to ZooKeeper ensemble=127.0.0.1:57490 2024-11-23T22:49:12,593 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:12,595 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:12,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:458090x0, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T22:49:12,618 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:458090x0, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:49:12,619 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T22:49:12,624 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45809-0x10169e2f9690003 connected 2024-11-23T22:49:12,630 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T22:49:12,631 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T22:49:12,634 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T22:49:12,636 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45809 2024-11-23T22:49:12,637 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45809 2024-11-23T22:49:12,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45809 2024-11-23T22:49:12,650 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45809 2024-11-23T22:49:12,650 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45809 2024-11-23T22:49:12,673 DEBUG [M:0;dd96e8dc8369:32885 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;dd96e8dc8369:32885 2024-11-23T22:49:12,676 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/dd96e8dc8369,32885,1732402151433 2024-11-23T22:49:12,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:49:12,692 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:49:12,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:49:12,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:49:12,698 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/dd96e8dc8369,32885,1732402151433 2024-11-23T22:49:12,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T22:49:12,733 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T22:49:12,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:12,733 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:12,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:12,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T22:49:12,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:12,736 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T22:49:12,739 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/dd96e8dc8369,32885,1732402151433 from backup master directory 2024-11-23T22:49:12,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:49:12,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:49:12,754 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:49:12,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/dd96e8dc8369,32885,1732402151433 2024-11-23T22:49:12,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:49:12,756 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T22:49:12,757 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=dd96e8dc8369,32885,1732402151433 2024-11-23T22:49:12,759 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-23T22:49:12,761 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-23T22:49:12,833 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/hbase.id] with ID: 38235187-828f-448d-9ed3-008fffa813da 2024-11-23T22:49:12,834 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/.tmp/hbase.id 2024-11-23T22:49:12,842 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:12,842 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:12,854 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_801357093_22 at /127.0.0.1:58786 [Receiving block BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775776_1003] {}] datanode.DataXceiver(331): 127.0.0.1:36219:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58786 dst: /127.0.0.1:36219 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:12,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36219 is added to blk_-9223372036854775776_1004 (size=42) 2024-11-23T22:49:12,864 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-23T22:49:12,864 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/.tmp/hbase.id]:[hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/hbase.id] 2024-11-23T22:49:12,915 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:12,921 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T22:49:12,947 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-23T22:49:12,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:12,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:12,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:12,955 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:12,976 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:12,976 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:12,985 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_801357093_22 at /127.0.0.1:35000 [Receiving block BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775760_1005] {}] datanode.DataXceiver(331): 127.0.0.1:43763:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35000 dst: /127.0.0.1:43763 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:12,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43763 is added to blk_-9223372036854775760_1006 (size=196) 2024-11-23T22:49:13,401 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-23T22:49:13,417 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T22:49:13,419 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T22:49:13,424 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T22:49:13,457 WARN [IPC Server handler 4 on default port 40647 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-23T22:49:13,458 WARN [IPC Server handler 4 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-23T22:49:13,458 WARN [IPC Server handler 4 on default port 40647 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-23T22:49:13,458 WARN [IPC Server handler 4 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-23T22:49:13,462 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(635): Failed write hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/.tabledesc/.tableinfo.0000000001.1189; retrying up to 10 times org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/.tabledesc/.tableinfo.0000000001.1189 could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.FSTableDescriptors.writeTableDescriptor(FSTableDescriptors.java:631) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FSTableDescriptors.createTableDescriptorForTableDirectory(FSTableDescriptors.java:707) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.region.MasterRegion.bootstrap(MasterRegion.java:241) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:410) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:13,472 WARN [IPC Server handler 1 on default port 40647 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-23T22:49:13,473 WARN [IPC Server handler 1 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-23T22:49:13,473 WARN [IPC Server handler 1 on default port 40647 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-23T22:49:13,473 WARN [IPC Server handler 1 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-23T22:49:13,475 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(635): Failed write hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/.tabledesc/.tableinfo.0000000002.1189; retrying up to 10 times org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/.tabledesc/.tableinfo.0000000002.1189 could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.FSTableDescriptors.writeTableDescriptor(FSTableDescriptors.java:631) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FSTableDescriptors.createTableDescriptorForTableDirectory(FSTableDescriptors.java:707) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.region.MasterRegion.bootstrap(MasterRegion.java:241) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:410) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:13,481 WARN [IPC Server handler 0 on default port 40647 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-23T22:49:13,481 WARN [IPC Server handler 0 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-23T22:49:13,481 WARN [IPC Server handler 0 on default port 40647 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-23T22:49:13,481 WARN [IPC Server handler 0 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-23T22:49:13,482 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(635): Failed write hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/.tabledesc/.tableinfo.0000000003.1189; retrying up to 10 times org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/.tabledesc/.tableinfo.0000000003.1189 could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.FSTableDescriptors.writeTableDescriptor(FSTableDescriptors.java:631) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FSTableDescriptors.createTableDescriptorForTableDirectory(FSTableDescriptors.java:707) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.region.MasterRegion.bootstrap(MasterRegion.java:241) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:410) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:13,489 WARN [IPC Server handler 2 on default port 40647 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-23T22:49:13,489 WARN [IPC Server handler 2 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-23T22:49:13,489 WARN [IPC Server handler 2 on default port 40647 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-23T22:49:13,490 WARN [IPC Server handler 2 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-23T22:49:13,491 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(635): Failed write hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/.tabledesc/.tableinfo.0000000004.1189; retrying up to 10 times org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/.tabledesc/.tableinfo.0000000004.1189 could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.FSTableDescriptors.writeTableDescriptor(FSTableDescriptors.java:631) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FSTableDescriptors.createTableDescriptorForTableDirectory(FSTableDescriptors.java:707) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.region.MasterRegion.bootstrap(MasterRegion.java:241) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:410) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:13,503 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:13,503 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:13,520 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_801357093_22 at /127.0.0.1:58812 [Receiving block BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775744_1007] {}] datanode.DataXceiver(331): 127.0.0.1:36219:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58812 dst: /127.0.0.1:36219 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:13,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36219 is added to blk_-9223372036854775744_1008 (size=1189) 2024-11-23T22:49:13,528 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-23T22:49:13,531 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/.tabledesc/.tableinfo.0000000001.1189 2024-11-23T22:49:13,533 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/.tabledesc/.tableinfo.0000000002.1189 2024-11-23T22:49:13,534 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/.tabledesc/.tableinfo.0000000003.1189 2024-11-23T22:49:13,536 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/.tabledesc/.tableinfo.0000000004.1189 2024-11-23T22:49:13,552 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store 2024-11-23T22:49:13,578 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:13,578 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:13,583 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_801357093_22 at /127.0.0.1:58826 [Receiving block BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775728_1009] {}] datanode.DataXceiver(331): 127.0.0.1:36219:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58826 dst: /127.0.0.1:36219 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:13,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36219 is added to blk_-9223372036854775728_1010 (size=34) 2024-11-23T22:49:13,590 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-23T22:49:13,595 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-23T22:49:13,599 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:49:13,601 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T22:49:13,601 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:49:13,601 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:49:13,603 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T22:49:13,603 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:49:13,603 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:49:13,605 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732402153601Disabling compacts and flushes for region at 1732402153601Disabling writes for close at 1732402153603 (+2 ms)Writing region close event to WAL at 1732402153603Closed at 1732402153603 2024-11-23T22:49:13,607 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/.initializing 2024-11-23T22:49:13,607 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/WALs/dd96e8dc8369,32885,1732402151433 2024-11-23T22:49:13,617 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T22:49:13,635 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C32885%2C1732402151433, suffix=, logDir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/WALs/dd96e8dc8369,32885,1732402151433, archiveDir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/oldWALs, maxLogs=10 2024-11-23T22:49:13,675 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/WALs/dd96e8dc8369,32885,1732402151433/dd96e8dc8369%2C32885%2C1732402151433.1732402153641, exclude list is [], retry=0 2024-11-23T22:49:13,678 WARN [IPC Server handler 4 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-23T22:49:13,678 WARN [IPC Server handler 4 on default port 40647 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-23T22:49:13,679 WARN [IPC Server handler 4 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-23T22:49:13,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:13,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43763,DS-5999b11b-fe98-468f-bf4c-ea395a07b98b,DISK] 2024-11-23T22:49:13,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35543,DS-c9e3d8d2-4dea-4625-b764-06ae2b7c1391,DISK] 2024-11-23T22:49:13,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-23T22:49:13,747 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/WALs/dd96e8dc8369,32885,1732402151433/dd96e8dc8369%2C32885%2C1732402151433.1732402153641 2024-11-23T22:49:13,748 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43121:43121),(127.0.0.1/127.0.0.1:42995:42995)] 2024-11-23T22:49:13,749 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T22:49:13,749 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:49:13,753 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:13,754 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:13,793 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:13,819 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T22:49:13,823 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:13,825 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:49:13,826 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:13,829 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T22:49:13,830 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:13,831 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T22:49:13,832 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:13,835 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T22:49:13,835 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:13,836 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T22:49:13,837 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:13,840 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T22:49:13,840 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:13,841 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T22:49:13,841 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:13,845 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:13,846 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:13,851 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:13,852 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:13,856 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T22:49:13,860 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:13,867 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T22:49:13,868 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62268414, jitterRate=-0.07212832570075989}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T22:49:13,876 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732402153767Initializing all the Stores at 1732402153769 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402153770 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402153770Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402153770Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402153770Cleaning up temporary data from old regions at 1732402153852 (+82 ms)Region opened successfully at 1732402153876 (+24 ms) 2024-11-23T22:49:13,877 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T22:49:13,917 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bfc7440, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0 2024-11-23T22:49:13,955 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T22:49:13,970 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T22:49:13,970 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T22:49:13,973 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T22:49:13,975 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-23T22:49:13,979 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-23T22:49:13,979 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T22:49:14,012 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T22:49:14,022 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T22:49:14,074 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=2) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-23T22:49:14,074 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-23T22:49:14,074 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=1) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-23T22:49:14,074 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-23T22:49:14,075 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-23T22:49:14,075 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-23T22:49:14,100 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T22:49:14,103 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T22:49:14,104 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T22:49:14,112 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T22:49:14,114 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T22:49:14,118 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T22:49:14,122 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T22:49:14,124 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T22:49:14,132 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T22:49:14,154 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T22:49:14,164 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T22:49:14,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T22:49:14,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T22:49:14,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T22:49:14,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:14,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:14,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:14,176 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T22:49:14,176 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:14,182 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=dd96e8dc8369,32885,1732402151433, sessionid=0x10169e2f9690000, setting cluster-up flag (Was=false) 2024-11-23T22:49:14,217 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:14,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:14,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:14,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:14,249 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T22:49:14,251 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,32885,1732402151433 2024-11-23T22:49:14,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:14,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:14,270 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:14,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:14,301 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T22:49:14,303 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,32885,1732402151433 2024-11-23T22:49:14,311 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T22:49:14,356 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer(746): ClusterId : 38235187-828f-448d-9ed3-008fffa813da 2024-11-23T22:49:14,356 INFO [RS:1;dd96e8dc8369:35195 {}] regionserver.HRegionServer(746): ClusterId : 38235187-828f-448d-9ed3-008fffa813da 2024-11-23T22:49:14,357 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.HRegionServer(746): ClusterId : 38235187-828f-448d-9ed3-008fffa813da 2024-11-23T22:49:14,359 DEBUG [RS:0;dd96e8dc8369:38153 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T22:49:14,359 DEBUG [RS:1;dd96e8dc8369:35195 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T22:49:14,359 DEBUG [RS:2;dd96e8dc8369:45809 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T22:49:14,387 DEBUG [RS:1;dd96e8dc8369:35195 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T22:49:14,387 DEBUG [RS:2;dd96e8dc8369:45809 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T22:49:14,387 DEBUG [RS:1;dd96e8dc8369:35195 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T22:49:14,387 DEBUG [RS:2;dd96e8dc8369:45809 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T22:49:14,388 DEBUG [RS:0;dd96e8dc8369:38153 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T22:49:14,388 DEBUG [RS:0;dd96e8dc8369:38153 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T22:49:14,397 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T22:49:14,408 DEBUG [RS:1;dd96e8dc8369:35195 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T22:49:14,408 DEBUG [RS:2;dd96e8dc8369:45809 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T22:49:14,409 DEBUG [RS:1;dd96e8dc8369:35195 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12c1bd10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0 2024-11-23T22:49:14,409 DEBUG [RS:2;dd96e8dc8369:45809 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21bdae5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0 2024-11-23T22:49:14,409 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T22:49:14,411 DEBUG [RS:0;dd96e8dc8369:38153 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T22:49:14,411 DEBUG [RS:0;dd96e8dc8369:38153 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ec66fb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0 2024-11-23T22:49:14,418 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T22:49:14,425 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: dd96e8dc8369,32885,1732402151433 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T22:49:14,435 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5 2024-11-23T22:49:14,436 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5 2024-11-23T22:49:14,436 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5 2024-11-23T22:49:14,436 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5 2024-11-23T22:49:14,436 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/dd96e8dc8369:0, corePoolSize=10, maxPoolSize=10 2024-11-23T22:49:14,436 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,436 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2 2024-11-23T22:49:14,437 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,443 DEBUG [RS:1;dd96e8dc8369:35195 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;dd96e8dc8369:35195 2024-11-23T22:49:14,446 DEBUG [RS:0;dd96e8dc8369:38153 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;dd96e8dc8369:38153 2024-11-23T22:49:14,449 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T22:49:14,449 INFO [RS:1;dd96e8dc8369:35195 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T22:49:14,449 INFO [RS:1;dd96e8dc8369:35195 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T22:49:14,449 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T22:49:14,450 DEBUG [RS:1;dd96e8dc8369:35195 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T22:49:14,450 DEBUG [RS:0;dd96e8dc8369:38153 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T22:49:14,453 INFO [RS:1;dd96e8dc8369:35195 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,32885,1732402151433 with port=35195, startcode=1732402152524 2024-11-23T22:49:14,453 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,32885,1732402151433 with port=38153, startcode=1732402152392 2024-11-23T22:49:14,455 DEBUG [RS:2;dd96e8dc8369:45809 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;dd96e8dc8369:45809 2024-11-23T22:49:14,455 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T22:49:14,455 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T22:49:14,456 DEBUG [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T22:49:14,459 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,32885,1732402151433 with port=45809, startcode=1732402152586 2024-11-23T22:49:14,466 DEBUG [RS:1;dd96e8dc8369:35195 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T22:49:14,466 DEBUG [RS:0;dd96e8dc8369:38153 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T22:49:14,466 DEBUG [RS:2;dd96e8dc8369:45809 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T22:49:14,473 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T22:49:14,474 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T22:49:14,484 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:14,484 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T22:49:14,512 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732402184512 2024-11-23T22:49:14,514 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T22:49:14,516 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T22:49:14,520 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T22:49:14,521 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T22:49:14,521 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T22:49:14,521 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T22:49:14,538 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45301, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T22:49:14,538 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43933, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T22:49:14,538 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38163, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T22:49:14,536 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,546 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32885 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T22:49:14,549 WARN [IPC Server handler 3 on default port 40647 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-23T22:49:14,550 WARN [IPC Server handler 3 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-23T22:49:14,550 WARN [IPC Server handler 3 on default port 40647 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-23T22:49:14,550 WARN [IPC Server handler 3 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-23T22:49:14,552 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32885 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T22:49:14,552 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(635): Failed write hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321; retrying up to 10 times org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.FSTableDescriptors.writeTableDescriptor(FSTableDescriptors.java:631) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FSTableDescriptors.tryUpdateAndGetMetaTableDescriptor(FSTableDescriptors.java:159) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.InitMetaProcedure.writeFsLayout(InitMetaProcedure.java:87) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.InitMetaProcedure.executeFromState(InitMetaProcedure.java:103) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.InitMetaProcedure.executeFromState(InitMetaProcedure.java:55) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T22:49:14,553 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32885 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T22:49:14,560 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T22:49:14,563 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T22:49:14,563 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T22:49:14,568 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T22:49:14,569 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T22:49:14,576 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402154570,5,FailOnTimeoutGroup] 2024-11-23T22:49:14,577 DEBUG [RS:1;dd96e8dc8369:35195 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-23T22:49:14,577 DEBUG [RS:0;dd96e8dc8369:38153 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-23T22:49:14,577 WARN [RS:0;dd96e8dc8369:38153 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-23T22:49:14,577 WARN [RS:1;dd96e8dc8369:35195 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-23T22:49:14,577 DEBUG [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-23T22:49:14,577 WARN [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-23T22:49:14,580 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402154576,5,FailOnTimeoutGroup] 2024-11-23T22:49:14,580 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,580 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T22:49:14,582 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,582 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,587 WARN [PEWorker-1 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:14,587 WARN [PEWorker-1 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:14,601 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_801357093_22 at /127.0.0.1:58854 [Receiving block BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775712_1012] {}] datanode.DataXceiver(331): 127.0.0.1:36219:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58854 dst: /127.0.0.1:36219 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:14,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36219 is added to blk_-9223372036854775712_1013 (size=1321) 2024-11-23T22:49:14,611 WARN [PEWorker-1 {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-23T22:49:14,614 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T22:49:14,614 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/.tabledesc/.tableinfo.0000000002.1321 2024-11-23T22:49:14,615 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef 2024-11-23T22:49:14,632 WARN [IPC Server handler 4 on default port 40647 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-23T22:49:14,632 WARN [IPC Server handler 4 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-23T22:49:14,632 WARN [IPC Server handler 4 on default port 40647 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-23T22:49:14,633 WARN [IPC Server handler 4 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-23T22:49:14,634 WARN [PEWorker-1 {}] procedure.InitMetaProcedure(132): Failed to init meta, suspend 1000secs org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740/.regioninfo could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.writeRegionInfoFileContent(HRegionFileSystem.java:815) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.writeRegionInfoOnFilesystem(HRegionFileSystem.java:906) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.writeRegionInfoOnFilesystem(HRegionFileSystem.java:868) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.createRegionOnFileSystem(HRegionFileSystem.java:936) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.createRegionDir(HRegion.java:7606) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.createHRegion(HRegion.java:7574) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.createHRegion(HRegion.java:7557) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.createHRegion(HRegion.java:7612) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.InitMetaProcedure.writeFsLayout(InitMetaProcedure.java:89) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.InitMetaProcedure.executeFromState(InitMetaProcedure.java:103) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.InitMetaProcedure.executeFromState(InitMetaProcedure.java:55) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T22:49:14,635 INFO [PEWorker-1 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=1, state=WAITING_TIMEOUT:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta; timeout=1000, timestamp=1732402155635 2024-11-23T22:49:14,679 INFO [RS:1;dd96e8dc8369:35195 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,32885,1732402151433 with port=35195, startcode=1732402152524 2024-11-23T22:49:14,679 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,32885,1732402151433 with port=45809, startcode=1732402152586 2024-11-23T22:49:14,679 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,32885,1732402151433 with port=38153, startcode=1732402152392 2024-11-23T22:49:14,681 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32885 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd96e8dc8369,35195,1732402152524 2024-11-23T22:49:14,684 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32885 {}] master.ServerManager(517): Registering regionserver=dd96e8dc8369,35195,1732402152524 2024-11-23T22:49:14,692 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32885 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd96e8dc8369,38153,1732402152392 2024-11-23T22:49:14,692 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32885 {}] master.ServerManager(517): Registering regionserver=dd96e8dc8369,38153,1732402152392 2024-11-23T22:49:14,692 DEBUG [RS:1;dd96e8dc8369:35195 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef 2024-11-23T22:49:14,693 DEBUG [RS:1;dd96e8dc8369:35195 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40647 2024-11-23T22:49:14,693 DEBUG [RS:1;dd96e8dc8369:35195 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T22:49:14,698 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32885 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd96e8dc8369,45809,1732402152586 2024-11-23T22:49:14,698 DEBUG [RS:0;dd96e8dc8369:38153 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef 2024-11-23T22:49:14,698 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32885 {}] master.ServerManager(517): Registering regionserver=dd96e8dc8369,45809,1732402152586 2024-11-23T22:49:14,698 DEBUG [RS:0;dd96e8dc8369:38153 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40647 2024-11-23T22:49:14,698 DEBUG [RS:0;dd96e8dc8369:38153 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T22:49:14,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T22:49:14,703 DEBUG [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef 2024-11-23T22:49:14,703 DEBUG [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40647 2024-11-23T22:49:14,703 DEBUG [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T22:49:14,831 DEBUG [RS:1;dd96e8dc8369:35195 {}] zookeeper.ZKUtil(111): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd96e8dc8369,35195,1732402152524 2024-11-23T22:49:14,831 WARN [RS:1;dd96e8dc8369:35195 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T22:49:14,831 INFO [RS:1;dd96e8dc8369:35195 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T22:49:14,831 DEBUG [RS:1;dd96e8dc8369:35195 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/WALs/dd96e8dc8369,35195,1732402152524 2024-11-23T22:49:14,849 DEBUG [RS:0;dd96e8dc8369:38153 {}] zookeeper.ZKUtil(111): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd96e8dc8369,38153,1732402152392 2024-11-23T22:49:14,849 WARN [RS:0;dd96e8dc8369:38153 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T22:49:14,849 DEBUG [RS:2;dd96e8dc8369:45809 {}] zookeeper.ZKUtil(111): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd96e8dc8369,45809,1732402152586 2024-11-23T22:49:14,849 WARN [RS:2;dd96e8dc8369:45809 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T22:49:14,849 INFO [RS:0;dd96e8dc8369:38153 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T22:49:14,849 INFO [RS:2;dd96e8dc8369:45809 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T22:49:14,850 DEBUG [RS:0;dd96e8dc8369:38153 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/WALs/dd96e8dc8369,38153,1732402152392 2024-11-23T22:49:14,850 DEBUG [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/WALs/dd96e8dc8369,45809,1732402152586 2024-11-23T22:49:14,851 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd96e8dc8369,38153,1732402152392] 2024-11-23T22:49:14,851 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd96e8dc8369,45809,1732402152586] 2024-11-23T22:49:14,851 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd96e8dc8369,35195,1732402152524] 2024-11-23T22:49:14,874 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T22:49:14,874 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T22:49:14,874 INFO [RS:1;dd96e8dc8369:35195 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T22:49:14,892 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T22:49:14,892 INFO [RS:1;dd96e8dc8369:35195 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T22:49:14,896 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T22:49:14,900 INFO [RS:1;dd96e8dc8369:35195 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T22:49:14,900 INFO [RS:1;dd96e8dc8369:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,900 INFO [RS:0;dd96e8dc8369:38153 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T22:49:14,900 INFO [RS:0;dd96e8dc8369:38153 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,904 INFO [RS:2;dd96e8dc8369:45809 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T22:49:14,905 INFO [RS:2;dd96e8dc8369:45809 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,905 INFO [RS:1;dd96e8dc8369:35195 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T22:49:14,910 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T22:49:14,910 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T22:49:14,912 INFO [RS:2;dd96e8dc8369:45809 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T22:49:14,912 INFO [RS:0;dd96e8dc8369:38153 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T22:49:14,912 INFO [RS:1;dd96e8dc8369:35195 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T22:49:14,914 INFO [RS:1;dd96e8dc8369:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,914 INFO [RS:0;dd96e8dc8369:38153 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,914 INFO [RS:2;dd96e8dc8369:45809 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,914 DEBUG [RS:0;dd96e8dc8369:38153 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,914 DEBUG [RS:2;dd96e8dc8369:45809 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,914 DEBUG [RS:0;dd96e8dc8369:38153 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,914 DEBUG [RS:2;dd96e8dc8369:45809 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,915 DEBUG [RS:2;dd96e8dc8369:45809 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,915 DEBUG [RS:0;dd96e8dc8369:38153 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,915 DEBUG [RS:2;dd96e8dc8369:45809 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,915 DEBUG [RS:0;dd96e8dc8369:38153 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,915 DEBUG [RS:0;dd96e8dc8369:38153 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,915 DEBUG [RS:2;dd96e8dc8369:45809 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,915 DEBUG [RS:0;dd96e8dc8369:38153 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2 2024-11-23T22:49:14,915 DEBUG [RS:2;dd96e8dc8369:45809 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2 2024-11-23T22:49:14,915 DEBUG [RS:0;dd96e8dc8369:38153 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,915 DEBUG [RS:2;dd96e8dc8369:45809 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,915 DEBUG [RS:2;dd96e8dc8369:45809 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,915 DEBUG [RS:0;dd96e8dc8369:38153 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,916 DEBUG [RS:2;dd96e8dc8369:45809 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,916 DEBUG [RS:0;dd96e8dc8369:38153 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,916 DEBUG [RS:2;dd96e8dc8369:45809 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,916 DEBUG [RS:0;dd96e8dc8369:38153 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,916 DEBUG [RS:0;dd96e8dc8369:38153 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,916 DEBUG [RS:2;dd96e8dc8369:45809 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,916 DEBUG [RS:2;dd96e8dc8369:45809 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,916 DEBUG [RS:1;dd96e8dc8369:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,916 DEBUG [RS:0;dd96e8dc8369:38153 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,916 DEBUG [RS:2;dd96e8dc8369:45809 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:49:14,916 DEBUG [RS:1;dd96e8dc8369:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,916 DEBUG [RS:0;dd96e8dc8369:38153 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:49:14,916 DEBUG [RS:2;dd96e8dc8369:45809 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:49:14,916 DEBUG [RS:1;dd96e8dc8369:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,916 DEBUG [RS:0;dd96e8dc8369:38153 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:49:14,916 DEBUG [RS:1;dd96e8dc8369:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,917 DEBUG [RS:1;dd96e8dc8369:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,917 DEBUG [RS:1;dd96e8dc8369:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2 2024-11-23T22:49:14,917 DEBUG [RS:1;dd96e8dc8369:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,917 DEBUG [RS:1;dd96e8dc8369:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,917 DEBUG [RS:1;dd96e8dc8369:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,917 DEBUG [RS:1;dd96e8dc8369:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,917 DEBUG [RS:1;dd96e8dc8369:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,918 DEBUG [RS:1;dd96e8dc8369:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:14,918 DEBUG [RS:1;dd96e8dc8369:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:49:14,918 DEBUG [RS:1;dd96e8dc8369:35195 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:49:14,923 INFO [RS:2;dd96e8dc8369:45809 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,923 INFO [RS:2;dd96e8dc8369:45809 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,924 INFO [RS:2;dd96e8dc8369:45809 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,924 INFO [RS:2;dd96e8dc8369:45809 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,924 INFO [RS:2;dd96e8dc8369:45809 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,924 INFO [RS:2;dd96e8dc8369:45809 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,45809,1732402152586-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T22:49:14,927 INFO [RS:0;dd96e8dc8369:38153 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,927 INFO [RS:1;dd96e8dc8369:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,927 INFO [RS:0;dd96e8dc8369:38153 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,927 INFO [RS:0;dd96e8dc8369:38153 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,927 INFO [RS:1;dd96e8dc8369:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,927 INFO [RS:0;dd96e8dc8369:38153 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,927 INFO [RS:1;dd96e8dc8369:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,927 INFO [RS:0;dd96e8dc8369:38153 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,927 INFO [RS:1;dd96e8dc8369:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,927 INFO [RS:0;dd96e8dc8369:38153 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,38153,1732402152392-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T22:49:14,927 INFO [RS:1;dd96e8dc8369:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,927 INFO [RS:1;dd96e8dc8369:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,35195,1732402152524-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T22:49:14,955 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T22:49:14,956 INFO [RS:1;dd96e8dc8369:35195 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T22:49:14,957 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T22:49:14,958 INFO [RS:0;dd96e8dc8369:38153 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,38153,1732402152392-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,958 INFO [RS:2;dd96e8dc8369:45809 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,45809,1732402152586-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,958 INFO [RS:1;dd96e8dc8369:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,35195,1732402152524-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,958 INFO [RS:0;dd96e8dc8369:38153 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,958 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.Replication(171): dd96e8dc8369,38153,1732402152392 started 2024-11-23T22:49:14,958 INFO [RS:2;dd96e8dc8369:45809 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,958 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.Replication(171): dd96e8dc8369,45809,1732402152586 started 2024-11-23T22:49:14,958 INFO [RS:1;dd96e8dc8369:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,960 INFO [RS:1;dd96e8dc8369:35195 {}] regionserver.Replication(171): dd96e8dc8369,35195,1732402152524 started 2024-11-23T22:49:14,986 INFO [RS:2;dd96e8dc8369:45809 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,987 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer(1482): Serving as dd96e8dc8369,45809,1732402152586, RpcServer on dd96e8dc8369/172.17.0.2:45809, sessionid=0x10169e2f9690003 2024-11-23T22:49:14,988 DEBUG [RS:2;dd96e8dc8369:45809 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T22:49:14,988 DEBUG [RS:2;dd96e8dc8369:45809 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd96e8dc8369,45809,1732402152586 2024-11-23T22:49:14,988 DEBUG [RS:2;dd96e8dc8369:45809 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,45809,1732402152586' 2024-11-23T22:49:14,989 DEBUG [RS:2;dd96e8dc8369:45809 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T22:49:14,990 DEBUG [RS:2;dd96e8dc8369:45809 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T22:49:14,991 DEBUG [RS:2;dd96e8dc8369:45809 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T22:49:14,991 DEBUG [RS:2;dd96e8dc8369:45809 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T22:49:14,991 DEBUG [RS:2;dd96e8dc8369:45809 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd96e8dc8369,45809,1732402152586 2024-11-23T22:49:14,991 DEBUG [RS:2;dd96e8dc8369:45809 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,45809,1732402152586' 2024-11-23T22:49:14,991 DEBUG [RS:2;dd96e8dc8369:45809 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T22:49:14,991 INFO [RS:0;dd96e8dc8369:38153 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,991 INFO [RS:1;dd96e8dc8369:35195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:14,991 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.HRegionServer(1482): Serving as dd96e8dc8369,38153,1732402152392, RpcServer on dd96e8dc8369/172.17.0.2:38153, sessionid=0x10169e2f9690001 2024-11-23T22:49:14,992 DEBUG [RS:0;dd96e8dc8369:38153 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T22:49:14,992 INFO [RS:1;dd96e8dc8369:35195 {}] regionserver.HRegionServer(1482): Serving as dd96e8dc8369,35195,1732402152524, RpcServer on dd96e8dc8369/172.17.0.2:35195, sessionid=0x10169e2f9690002 2024-11-23T22:49:14,992 DEBUG [RS:0;dd96e8dc8369:38153 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd96e8dc8369,38153,1732402152392 2024-11-23T22:49:14,992 DEBUG [RS:1;dd96e8dc8369:35195 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T22:49:14,992 DEBUG [RS:1;dd96e8dc8369:35195 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd96e8dc8369,35195,1732402152524 2024-11-23T22:49:14,992 DEBUG [RS:1;dd96e8dc8369:35195 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,35195,1732402152524' 2024-11-23T22:49:14,992 DEBUG [RS:1;dd96e8dc8369:35195 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T22:49:14,992 DEBUG [RS:2;dd96e8dc8369:45809 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T22:49:14,992 DEBUG [RS:0;dd96e8dc8369:38153 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,38153,1732402152392' 2024-11-23T22:49:14,993 DEBUG [RS:0;dd96e8dc8369:38153 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T22:49:14,993 DEBUG [RS:1;dd96e8dc8369:35195 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T22:49:14,994 DEBUG [RS:2;dd96e8dc8369:45809 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T22:49:14,994 DEBUG [RS:1;dd96e8dc8369:35195 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T22:49:14,994 DEBUG [RS:1;dd96e8dc8369:35195 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T22:49:14,994 INFO [RS:2;dd96e8dc8369:45809 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T22:49:14,994 DEBUG [RS:1;dd96e8dc8369:35195 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd96e8dc8369,35195,1732402152524 2024-11-23T22:49:14,994 INFO [RS:2;dd96e8dc8369:45809 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T22:49:14,994 DEBUG [RS:1;dd96e8dc8369:35195 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,35195,1732402152524' 2024-11-23T22:49:14,995 DEBUG [RS:1;dd96e8dc8369:35195 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T22:49:14,996 DEBUG [RS:1;dd96e8dc8369:35195 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T22:49:14,997 DEBUG [RS:1;dd96e8dc8369:35195 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T22:49:14,997 INFO [RS:1;dd96e8dc8369:35195 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T22:49:14,997 INFO [RS:1;dd96e8dc8369:35195 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T22:49:14,993 DEBUG [RS:0;dd96e8dc8369:38153 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T22:49:14,998 DEBUG [RS:0;dd96e8dc8369:38153 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T22:49:14,998 DEBUG [RS:0;dd96e8dc8369:38153 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T22:49:14,998 DEBUG [RS:0;dd96e8dc8369:38153 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd96e8dc8369,38153,1732402152392 2024-11-23T22:49:14,998 DEBUG [RS:0;dd96e8dc8369:38153 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,38153,1732402152392' 2024-11-23T22:49:14,998 DEBUG [RS:0;dd96e8dc8369:38153 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T22:49:14,999 DEBUG [RS:0;dd96e8dc8369:38153 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T22:49:14,999 DEBUG [RS:0;dd96e8dc8369:38153 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T22:49:14,999 INFO [RS:0;dd96e8dc8369:38153 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T22:49:14,999 INFO [RS:0;dd96e8dc8369:38153 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T22:49:15,099 INFO [RS:2;dd96e8dc8369:45809 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T22:49:15,099 INFO [RS:1;dd96e8dc8369:35195 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T22:49:15,100 INFO [RS:0;dd96e8dc8369:38153 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T22:49:15,102 INFO [RS:1;dd96e8dc8369:35195 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C35195%2C1732402152524, suffix=, logDir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/WALs/dd96e8dc8369,35195,1732402152524, archiveDir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/oldWALs, maxLogs=32 2024-11-23T22:49:15,104 INFO [RS:2;dd96e8dc8369:45809 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C45809%2C1732402152586, suffix=, logDir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/WALs/dd96e8dc8369,45809,1732402152586, archiveDir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/oldWALs, maxLogs=32 2024-11-23T22:49:15,109 INFO [RS:0;dd96e8dc8369:38153 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C38153%2C1732402152392, suffix=, logDir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/WALs/dd96e8dc8369,38153,1732402152392, archiveDir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/oldWALs, maxLogs=32 2024-11-23T22:49:15,125 DEBUG [RS:1;dd96e8dc8369:35195 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/WALs/dd96e8dc8369,35195,1732402152524/dd96e8dc8369%2C35195%2C1732402152524.1732402155105, exclude list is [], retry=0 2024-11-23T22:49:15,130 DEBUG [RS:2;dd96e8dc8369:45809 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/WALs/dd96e8dc8369,45809,1732402152586/dd96e8dc8369%2C45809%2C1732402152586.1732402155107, exclude list is [], retry=0 2024-11-23T22:49:15,133 DEBUG [RS:0;dd96e8dc8369:38153 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/WALs/dd96e8dc8369,38153,1732402152392/dd96e8dc8369%2C38153%2C1732402152392.1732402155112, exclude list is [], retry=0 2024-11-23T22:49:15,133 WARN [IPC Server handler 4 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-23T22:49:15,133 WARN [IPC Server handler 4 on default port 40647 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-23T22:49:15,133 WARN [IPC Server handler 4 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-23T22:49:15,136 WARN [IPC Server handler 3 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-23T22:49:15,136 WARN [IPC Server handler 0 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-23T22:49:15,136 WARN [IPC Server handler 3 on default port 40647 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-23T22:49:15,136 WARN [IPC Server handler 0 on default port 40647 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-23T22:49:15,136 WARN [IPC Server handler 3 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-23T22:49:15,136 WARN [IPC Server handler 0 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-23T22:49:15,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43763,DS-5999b11b-fe98-468f-bf4c-ea395a07b98b,DISK] 2024-11-23T22:49:15,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35543,DS-c9e3d8d2-4dea-4625-b764-06ae2b7c1391,DISK] 2024-11-23T22:49:15,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43763,DS-5999b11b-fe98-468f-bf4c-ea395a07b98b,DISK] 2024-11-23T22:49:15,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35543,DS-c9e3d8d2-4dea-4625-b764-06ae2b7c1391,DISK] 2024-11-23T22:49:15,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43763,DS-5999b11b-fe98-468f-bf4c-ea395a07b98b,DISK] 2024-11-23T22:49:15,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35543,DS-c9e3d8d2-4dea-4625-b764-06ae2b7c1391,DISK] 2024-11-23T22:49:15,156 INFO [RS:2;dd96e8dc8369:45809 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/WALs/dd96e8dc8369,45809,1732402152586/dd96e8dc8369%2C45809%2C1732402152586.1732402155107 2024-11-23T22:49:15,166 DEBUG [RS:2;dd96e8dc8369:45809 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42995:42995),(127.0.0.1/127.0.0.1:43121:43121)] 2024-11-23T22:49:15,182 INFO [RS:1;dd96e8dc8369:35195 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/WALs/dd96e8dc8369,35195,1732402152524/dd96e8dc8369%2C35195%2C1732402152524.1732402155105 2024-11-23T22:49:15,184 INFO [RS:0;dd96e8dc8369:38153 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/WALs/dd96e8dc8369,38153,1732402152392/dd96e8dc8369%2C38153%2C1732402152392.1732402155112 2024-11-23T22:49:15,188 DEBUG [RS:1;dd96e8dc8369:35195 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43121:43121),(127.0.0.1/127.0.0.1:42995:42995)] 2024-11-23T22:49:15,188 DEBUG [RS:0;dd96e8dc8369:38153 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42995:42995),(127.0.0.1/127.0.0.1:43121:43121)] 2024-11-23T22:49:15,638 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T22:49:15,638 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T22:49:15,641 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:15,641 INFO [PEWorker-2 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T22:49:15,645 WARN [IPC Server handler 3 on default port 40647 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-23T22:49:15,646 WARN [IPC Server handler 3 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-23T22:49:15,646 WARN [IPC Server handler 3 on default port 40647 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-23T22:49:15,646 WARN [IPC Server handler 3 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-23T22:49:15,647 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(635): Failed write hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321; retrying up to 10 times org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.FSTableDescriptors.writeTableDescriptor(FSTableDescriptors.java:631) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FSTableDescriptors.tryUpdateAndGetMetaTableDescriptor(FSTableDescriptors.java:159) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.InitMetaProcedure.writeFsLayout(InitMetaProcedure.java:87) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.InitMetaProcedure.executeFromState(InitMetaProcedure.java:103) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.InitMetaProcedure.executeFromState(InitMetaProcedure.java:55) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T22:49:15,652 WARN [IPC Server handler 1 on default port 40647 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-23T22:49:15,652 WARN [IPC Server handler 1 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-23T22:49:15,652 WARN [IPC Server handler 1 on default port 40647 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-23T22:49:15,653 WARN [IPC Server handler 1 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-23T22:49:15,653 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(635): Failed write hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/.tabledesc/.tableinfo.0000000002.1321; retrying up to 10 times org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/.tabledesc/.tableinfo.0000000002.1321 could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.FSTableDescriptors.writeTableDescriptor(FSTableDescriptors.java:631) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FSTableDescriptors.tryUpdateAndGetMetaTableDescriptor(FSTableDescriptors.java:159) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.InitMetaProcedure.writeFsLayout(InitMetaProcedure.java:87) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.InitMetaProcedure.executeFromState(InitMetaProcedure.java:103) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.InitMetaProcedure.executeFromState(InitMetaProcedure.java:55) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T22:49:15,660 WARN [PEWorker-2 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:15,660 WARN [PEWorker-2 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:15,664 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_801357093_22 at /127.0.0.1:58862 [Receiving block BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775696_1017] {}] datanode.DataXceiver(331): 127.0.0.1:36219:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58862 dst: /127.0.0.1:36219 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:15,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36219 is added to blk_-9223372036854775696_1018 (size=1321) 2024-11-23T22:49:15,669 WARN [PEWorker-2 {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-23T22:49:15,672 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T22:49:15,673 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/.tabledesc/.tableinfo.0000000002.1321 2024-11-23T22:49:15,673 INFO [PEWorker-2 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/.tabledesc/.tableinfo.0000000003.1321 2024-11-23T22:49:15,673 INFO [PEWorker-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef 2024-11-23T22:49:15,679 WARN [IPC Server handler 3 on default port 40647 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-23T22:49:15,679 WARN [IPC Server handler 3 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-23T22:49:15,679 WARN [IPC Server handler 3 on default port 40647 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-23T22:49:15,679 WARN [IPC Server handler 3 on default port 40647 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-23T22:49:15,680 WARN [PEWorker-2 {}] procedure.InitMetaProcedure(132): Failed to init meta, suspend 2000secs org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740/.regioninfo could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.writeRegionInfoFileContent(HRegionFileSystem.java:815) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.writeRegionInfoOnFilesystem(HRegionFileSystem.java:906) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.writeRegionInfoOnFilesystem(HRegionFileSystem.java:868) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.createRegionOnFileSystem(HRegionFileSystem.java:936) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.createRegionDir(HRegion.java:7606) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.createHRegion(HRegion.java:7574) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.createHRegion(HRegion.java:7557) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.createHRegion(HRegion.java:7612) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.InitMetaProcedure.writeFsLayout(InitMetaProcedure.java:89) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.InitMetaProcedure.executeFromState(InitMetaProcedure.java:103) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.InitMetaProcedure.executeFromState(InitMetaProcedure.java:55) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T22:49:15,681 INFO [PEWorker-2 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=1, state=WAITING_TIMEOUT:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta; timeout=2000, timestamp=1732402157681 2024-11-23T22:49:16,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43763 is added to blk_-9223372036854775773_1004 (size=42) 2024-11-23T22:49:16,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35543 is added to blk_-9223372036854775772_1004 (size=42) 2024-11-23T22:49:16,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43763 is added to blk_-9223372036854775789_1002 (size=7) 2024-11-23T22:49:16,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35543 is added to blk_-9223372036854775788_1002 (size=7) 2024-11-23T22:49:16,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35543 is added to blk_-9223372036854775757_1006 (size=196) 2024-11-23T22:49:16,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35543 is added to blk_-9223372036854775741_1008 (size=1189) 2024-11-23T22:49:16,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43763 is added to blk_-9223372036854775740_1008 (size=1189) 2024-11-23T22:49:17,683 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T22:49:17,683 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T22:49:17,687 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:17,687 INFO [PEWorker-3 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T22:49:17,698 WARN [PEWorker-3 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:17,698 WARN [PEWorker-3 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:17,708 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_801357093_22 at /127.0.0.1:58890 [Receiving block BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775680_1019] {}] datanode.DataXceiver(331): 127.0.0.1:36219:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58890 dst: /127.0.0.1:36219 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:17,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36219 is added to blk_-9223372036854775680_1020 (size=1321) 2024-11-23T22:49:17,717 WARN [PEWorker-3 {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-23T22:49:17,718 INFO [PEWorker-3 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T22:49:17,718 INFO [PEWorker-3 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef 2024-11-23T22:49:17,724 WARN [PEWorker-3 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:17,724 WARN [PEWorker-3 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:17,733 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_801357093_22 at /127.0.0.1:59762 [Receiving block BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775664_1021] {}] datanode.DataXceiver(331): 127.0.0.1:35543:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59762 dst: /127.0.0.1:35543 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:17,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35543 is added to blk_-9223372036854775664_1022 (size=32) 2024-11-23T22:49:17,742 WARN [PEWorker-3 {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-23T22:49:17,743 DEBUG [PEWorker-3 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:49:17,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T22:49:17,748 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T22:49:17,748 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:17,750 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:49:17,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T22:49:17,755 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T22:49:17,755 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:17,756 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:49:17,756 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T22:49:17,760 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T22:49:17,760 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:17,761 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:49:17,761 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T22:49:17,764 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T22:49:17,764 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:17,765 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:49:17,766 DEBUG [PEWorker-3 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T22:49:17,767 DEBUG [PEWorker-3 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740 2024-11-23T22:49:17,768 DEBUG [PEWorker-3 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740 2024-11-23T22:49:17,772 DEBUG [PEWorker-3 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T22:49:17,772 DEBUG [PEWorker-3 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T22:49:17,773 DEBUG [PEWorker-3 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T22:49:17,777 DEBUG [PEWorker-3 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T22:49:17,798 DEBUG [PEWorker-3 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T22:49:17,799 INFO [PEWorker-3 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58917683, jitterRate=-0.12205810844898224}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T22:49:17,801 DEBUG [PEWorker-3 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732402157743Initializing all the Stores at 1732402157745 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402157745Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402157745Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402157745Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402157745Cleaning up temporary data from old regions at 1732402157772 (+27 ms)Region opened successfully at 1732402157801 (+29 ms) 2024-11-23T22:49:17,801 DEBUG [PEWorker-3 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T22:49:17,801 INFO [PEWorker-3 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T22:49:17,801 DEBUG [PEWorker-3 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T22:49:17,801 DEBUG [PEWorker-3 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T22:49:17,801 DEBUG [PEWorker-3 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T22:49:17,803 INFO [PEWorker-3 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T22:49:17,803 DEBUG [PEWorker-3 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732402157801Disabling compacts and flushes for region at 1732402157801Disabling writes for close at 1732402157801Writing region close event to WAL at 1732402157802 (+1 ms)Closed at 1732402157803 (+1 ms) 2024-11-23T22:49:17,806 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T22:49:17,806 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T22:49:17,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T22:49:17,826 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T22:49:17,829 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T22:49:17,982 DEBUG [dd96e8dc8369:32885 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-23T22:49:17,991 DEBUG [dd96e8dc8369:32885 {}] balancer.BalancerClusterState(204): Hosts are {dd96e8dc8369=0} racks are {/default-rack=0} 2024-11-23T22:49:17,998 DEBUG [dd96e8dc8369:32885 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T22:49:17,998 DEBUG [dd96e8dc8369:32885 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T22:49:17,998 DEBUG [dd96e8dc8369:32885 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-23T22:49:17,998 DEBUG [dd96e8dc8369:32885 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T22:49:17,998 DEBUG [dd96e8dc8369:32885 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T22:49:17,998 DEBUG [dd96e8dc8369:32885 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-23T22:49:17,998 INFO [dd96e8dc8369:32885 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T22:49:17,998 INFO [dd96e8dc8369:32885 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T22:49:17,998 INFO [dd96e8dc8369:32885 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-23T22:49:17,998 DEBUG [dd96e8dc8369:32885 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T22:49:18,007 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=dd96e8dc8369,45809,1732402152586 2024-11-23T22:49:18,013 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,45809,1732402152586, state=OPENING 2024-11-23T22:49:18,080 DEBUG [PEWorker-5 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T22:49:18,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:18,090 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:18,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:18,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:18,092 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:49:18,092 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:49:18,092 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:49:18,092 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:49:18,095 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T22:49:18,097 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,45809,1732402152586}] 2024-11-23T22:49:18,274 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T22:49:18,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35721, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T22:49:18,290 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T22:49:18,291 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T22:49:18,292 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T22:49:18,295 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C45809%2C1732402152586.meta, suffix=.meta, logDir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/WALs/dd96e8dc8369,45809,1732402152586, archiveDir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/oldWALs, maxLogs=32 2024-11-23T22:49:18,317 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/WALs/dd96e8dc8369,45809,1732402152586/dd96e8dc8369%2C45809%2C1732402152586.meta.1732402158298.meta, exclude list is [], retry=0 2024-11-23T22:49:18,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43763,DS-5999b11b-fe98-468f-bf4c-ea395a07b98b,DISK] 2024-11-23T22:49:18,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35543,DS-c9e3d8d2-4dea-4625-b764-06ae2b7c1391,DISK] 2024-11-23T22:49:18,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36219,DS-703608bf-71fd-4811-aa6f-e7553fea4492,DISK] 2024-11-23T22:49:18,329 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/WALs/dd96e8dc8369,45809,1732402152586/dd96e8dc8369%2C45809%2C1732402152586.meta.1732402158298.meta 2024-11-23T22:49:18,329 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42995:42995),(127.0.0.1/127.0.0.1:43121:43121),(127.0.0.1/127.0.0.1:36559:36559)] 2024-11-23T22:49:18,329 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T22:49:18,331 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T22:49:18,334 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T22:49:18,338 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T22:49:18,342 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T22:49:18,343 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:49:18,343 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T22:49:18,343 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T22:49:18,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T22:49:18,348 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T22:49:18,348 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:18,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:49:18,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T22:49:18,351 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T22:49:18,351 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:18,352 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:49:18,352 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T22:49:18,354 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T22:49:18,354 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:18,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:49:18,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T22:49:18,357 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T22:49:18,357 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:18,358 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:49:18,358 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T22:49:18,360 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740 2024-11-23T22:49:18,365 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740 2024-11-23T22:49:18,367 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T22:49:18,367 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T22:49:18,368 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T22:49:18,372 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T22:49:18,374 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62943356, jitterRate=-0.06207090616226196}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T22:49:18,374 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T22:49:18,377 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732402158344Writing region info on filesystem at 1732402158344Initializing all the Stores at 1732402158345 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402158345Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402158346 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402158346Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402158346Cleaning up temporary data from old regions at 1732402158367 (+21 ms)Running coprocessor post-open hooks at 1732402158374 (+7 ms)Region opened successfully at 1732402158377 (+3 ms) 2024-11-23T22:49:18,385 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732402158264 2024-11-23T22:49:18,397 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T22:49:18,398 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T22:49:18,400 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,45809,1732402152586 2024-11-23T22:49:18,402 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,45809,1732402152586, state=OPEN 2024-11-23T22:49:18,415 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T22:49:18,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T22:49:18,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T22:49:18,415 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:49:18,415 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:49:18,415 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:49:18,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T22:49:18,416 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:49:18,416 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=dd96e8dc8369,45809,1732402152586 2024-11-23T22:49:18,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T22:49:18,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,45809,1732402152586 in 319 msec 2024-11-23T22:49:18,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T22:49:18,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 611 msec 2024-11-23T22:49:18,434 DEBUG [PEWorker-4 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T22:49:18,434 INFO [PEWorker-4 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T22:49:18,482 DEBUG [PEWorker-4 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T22:49:18,484 DEBUG [PEWorker-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,45809,1732402152586, seqNum=-1] 2024-11-23T22:49:18,510 DEBUG [PEWorker-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T22:49:18,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45129, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T22:49:18,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 4.1980 sec 2024-11-23T22:49:18,548 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732402158547, completionTime=-1 2024-11-23T22:49:18,550 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-23T22:49:18,551 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T22:49:18,578 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-23T22:49:18,578 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732402218578 2024-11-23T22:49:18,578 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732402278578 2024-11-23T22:49:18,579 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-23T22:49:18,580 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-23T22:49:18,589 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,32885,1732402151433-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:18,590 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,32885,1732402151433-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:18,590 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,32885,1732402151433-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:18,592 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-dd96e8dc8369:32885, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:18,592 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:18,593 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:18,599 DEBUG [master/dd96e8dc8369:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T22:49:18,629 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 5.872sec 2024-11-23T22:49:18,633 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T22:49:18,635 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T22:49:18,636 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T22:49:18,636 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T22:49:18,637 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T22:49:18,638 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,32885,1732402151433-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T22:49:18,638 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,32885,1732402151433-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T22:49:18,644 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T22:49:18,645 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T22:49:18,646 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,32885,1732402151433-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:18,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46f2050c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T22:49:18,698 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-23T22:49:18,698 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-23T22:49:18,704 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request dd96e8dc8369,32885,-1 for getting cluster id 2024-11-23T22:49:18,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T22:49:18,722 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38235187-828f-448d-9ed3-008fffa813da' 2024-11-23T22:49:18,726 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T22:49:18,726 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38235187-828f-448d-9ed3-008fffa813da" 2024-11-23T22:49:18,727 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e0ccdbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T22:49:18,727 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [dd96e8dc8369,32885,-1] 2024-11-23T22:49:18,730 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T22:49:18,732 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:18,734 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T22:49:18,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28c511e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T22:49:18,738 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T22:49:18,747 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,45809,1732402152586, seqNum=-1] 2024-11-23T22:49:18,747 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T22:49:18,750 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T22:49:18,776 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=dd96e8dc8369,32885,1732402151433 2024-11-23T22:49:18,782 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T22:49:18,788 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is dd96e8dc8369,32885,1732402151433 2024-11-23T22:49:18,792 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4543bcec 2024-11-23T22:49:18,793 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T22:49:18,795 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T22:49:18,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestHBaseWalOnEC', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T22:49:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestHBaseWalOnEC 2024-11-23T22:49:18,814 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T22:49:18,818 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:18,820 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestHBaseWalOnEC" procId is: 4 2024-11-23T22:49:18,825 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T22:49:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T22:49:18,839 WARN [PEWorker-1 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:18,839 WARN [PEWorker-1 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:18,846 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_801357093_22 at /127.0.0.1:59780 [Receiving block BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775648_1024] {}] datanode.DataXceiver(331): 127.0.0.1:35543:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59780 dst: /127.0.0.1:35543 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:18,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35543 is added to blk_-9223372036854775648_1025 (size=392) 2024-11-23T22:49:18,854 WARN [PEWorker-1 {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-23T22:49:18,857 INFO [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c0f4f302e3b6b1e7ce6f6aaf333f0c7e, NAME => 'TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestHBaseWalOnEC', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef 2024-11-23T22:49:18,866 WARN [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:18,866 WARN [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:18,877 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_801357093_22 at /127.0.0.1:58930 [Receiving block BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775632_1026] {}] datanode.DataXceiver(331): 127.0.0.1:36219:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58930 dst: /127.0.0.1:36219 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:18,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36219 is added to blk_-9223372036854775632_1027 (size=51) 2024-11-23T22:49:18,884 WARN [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-23T22:49:18,885 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(898): Instantiated TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:49:18,885 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1722): Closing c0f4f302e3b6b1e7ce6f6aaf333f0c7e, disabling compactions & flushes 2024-11-23T22:49:18,885 INFO [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1755): Closing region TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e. 2024-11-23T22:49:18,885 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e. 2024-11-23T22:49:18,885 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e. after waiting 0 ms 2024-11-23T22:49:18,885 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e. 2024-11-23T22:49:18,885 INFO [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1973): Closed TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e. 2024-11-23T22:49:18,885 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1676): Region close journal for c0f4f302e3b6b1e7ce6f6aaf333f0c7e: Waiting for close lock at 1732402158885Disabling compacts and flushes for region at 1732402158885Disabling writes for close at 1732402158885Writing region close event to WAL at 1732402158885Closed at 1732402158885 2024-11-23T22:49:18,888 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T22:49:18,895 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e.","families":{"info":[{"qualifier":"regioninfo","vlen":50,"tag":[],"timestamp":"1732402158888"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732402158888"}]},"ts":"1732402158888"} 2024-11-23T22:49:18,903 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T22:49:18,905 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T22:49:18,908 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestHBaseWalOnEC","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732402158906"}]},"ts":"1732402158906"} 2024-11-23T22:49:18,913 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestHBaseWalOnEC, state=ENABLING in hbase:meta 2024-11-23T22:49:18,914 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {dd96e8dc8369=0} racks are {/default-rack=0} 2024-11-23T22:49:18,915 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T22:49:18,915 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T22:49:18,915 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-23T22:49:18,916 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T22:49:18,916 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T22:49:18,916 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-23T22:49:18,916 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T22:49:18,916 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T22:49:18,916 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-23T22:49:18,916 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T22:49:18,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=c0f4f302e3b6b1e7ce6f6aaf333f0c7e, ASSIGN}] 2024-11-23T22:49:18,919 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=c0f4f302e3b6b1e7ce6f6aaf333f0c7e, ASSIGN 2024-11-23T22:49:18,922 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=c0f4f302e3b6b1e7ce6f6aaf333f0c7e, ASSIGN; state=OFFLINE, location=dd96e8dc8369,38153,1732402152392; forceNewPlan=false, retain=false 2024-11-23T22:49:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T22:49:19,077 INFO [dd96e8dc8369:32885 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T22:49:19,078 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c0f4f302e3b6b1e7ce6f6aaf333f0c7e, regionState=OPENING, regionLocation=dd96e8dc8369,38153,1732402152392 2024-11-23T22:49:19,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=c0f4f302e3b6b1e7ce6f6aaf333f0c7e, ASSIGN because future has completed 2024-11-23T22:49:19,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0f4f302e3b6b1e7ce6f6aaf333f0c7e, server=dd96e8dc8369,38153,1732402152392}] 2024-11-23T22:49:19,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T22:49:19,241 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T22:49:19,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60961, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T22:49:19,252 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e. 2024-11-23T22:49:19,252 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c0f4f302e3b6b1e7ce6f6aaf333f0c7e, NAME => 'TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e.', STARTKEY => '', ENDKEY => ''} 2024-11-23T22:49:19,252 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestHBaseWalOnEC c0f4f302e3b6b1e7ce6f6aaf333f0c7e 2024-11-23T22:49:19,253 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:49:19,253 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c0f4f302e3b6b1e7ce6f6aaf333f0c7e 2024-11-23T22:49:19,253 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c0f4f302e3b6b1e7ce6f6aaf333f0c7e 2024-11-23T22:49:19,255 INFO [StoreOpener-c0f4f302e3b6b1e7ce6f6aaf333f0c7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region c0f4f302e3b6b1e7ce6f6aaf333f0c7e 2024-11-23T22:49:19,257 INFO [StoreOpener-c0f4f302e3b6b1e7ce6f6aaf333f0c7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0f4f302e3b6b1e7ce6f6aaf333f0c7e columnFamilyName cf 2024-11-23T22:49:19,257 DEBUG [StoreOpener-c0f4f302e3b6b1e7ce6f6aaf333f0c7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:19,258 INFO [StoreOpener-c0f4f302e3b6b1e7ce6f6aaf333f0c7e-1 {}] regionserver.HStore(327): Store=c0f4f302e3b6b1e7ce6f6aaf333f0c7e/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T22:49:19,258 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c0f4f302e3b6b1e7ce6f6aaf333f0c7e 2024-11-23T22:49:19,260 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/default/TestHBaseWalOnEC/c0f4f302e3b6b1e7ce6f6aaf333f0c7e 2024-11-23T22:49:19,260 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/default/TestHBaseWalOnEC/c0f4f302e3b6b1e7ce6f6aaf333f0c7e 2024-11-23T22:49:19,261 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c0f4f302e3b6b1e7ce6f6aaf333f0c7e 2024-11-23T22:49:19,261 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c0f4f302e3b6b1e7ce6f6aaf333f0c7e 2024-11-23T22:49:19,265 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c0f4f302e3b6b1e7ce6f6aaf333f0c7e 2024-11-23T22:49:19,270 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/default/TestHBaseWalOnEC/c0f4f302e3b6b1e7ce6f6aaf333f0c7e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T22:49:19,271 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c0f4f302e3b6b1e7ce6f6aaf333f0c7e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61235371, jitterRate=-0.08752186596393585}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T22:49:19,271 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0f4f302e3b6b1e7ce6f6aaf333f0c7e 2024-11-23T22:49:19,272 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c0f4f302e3b6b1e7ce6f6aaf333f0c7e: Running coprocessor pre-open hook at 1732402159253Writing region info on filesystem at 1732402159253Initializing all the Stores at 1732402159255 (+2 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402159255Cleaning up temporary data from old regions at 1732402159261 (+6 ms)Running coprocessor post-open hooks at 1732402159271 (+10 ms)Region opened successfully at 1732402159272 (+1 ms) 2024-11-23T22:49:19,274 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e., pid=6, masterSystemTime=1732402159241 2024-11-23T22:49:19,277 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e. 2024-11-23T22:49:19,277 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e. 2024-11-23T22:49:19,278 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c0f4f302e3b6b1e7ce6f6aaf333f0c7e, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,38153,1732402152392 2024-11-23T22:49:19,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0f4f302e3b6b1e7ce6f6aaf333f0c7e, server=dd96e8dc8369,38153,1732402152392 because future has completed 2024-11-23T22:49:19,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T22:49:19,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c0f4f302e3b6b1e7ce6f6aaf333f0c7e, server=dd96e8dc8369,38153,1732402152392 in 198 msec 2024-11-23T22:49:19,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T22:49:19,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=c0f4f302e3b6b1e7ce6f6aaf333f0c7e, ASSIGN in 370 msec 2024-11-23T22:49:19,292 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T22:49:19,292 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestHBaseWalOnEC","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732402159292"}]},"ts":"1732402159292"} 2024-11-23T22:49:19,295 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestHBaseWalOnEC, state=ENABLED in hbase:meta 2024-11-23T22:49:19,297 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T22:49:19,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestHBaseWalOnEC in 492 msec 2024-11-23T22:49:19,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43763 is added to blk_-9223372036854775725_1010 (size=34) 2024-11-23T22:49:19,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35543 is added to blk_-9223372036854775724_1010 (size=34) 2024-11-23T22:49:19,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36219 is added to blk_-9223372036854775756_1006 (size=196) 2024-11-23T22:49:19,430 WARN [Command processor {}] erasurecode.ErasureCodingWorker(145): No missing internal block. Skip reconstruction for task:BlockECReconstructionInfo( Recovering BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775696_1018 From: [127.0.0.1:36219] To: [[127.0.0.1:35543, 127.0.0.1:43763]) Block Indices: [0] 2024-11-23T22:49:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T22:49:19,457 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table TestHBaseWalOnEC get assigned. Timeout = 60000ms 2024-11-23T22:49:19,457 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestHBaseWalOnEC completed 2024-11-23T22:49:19,459 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T22:49:19,469 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table TestHBaseWalOnEC assigned to meta. Checking AM states. 2024-11-23T22:49:19,470 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T22:49:19,471 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table TestHBaseWalOnEC assigned. 2024-11-23T22:49:19,480 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestHBaseWalOnEC', row='row', locateType=CURRENT is [region=TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e., hostname=dd96e8dc8369,38153,1732402152392, seqNum=2] 2024-11-23T22:49:19,481 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T22:49:19,484 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T22:49:19,492 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestHBaseWalOnEC 2024-11-23T22:49:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC 2024-11-23T22:49:19,499 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC execute state=FLUSH_TABLE_PREPARE 2024-11-23T22:49:19,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T22:49:19,502 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T22:49:19,503 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T22:49:19,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T22:49:19,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38153 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T22:49:19,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e. 2024-11-23T22:49:19,674 INFO [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing c0f4f302e3b6b1e7ce6f6aaf333f0c7e 1/1 column families, dataSize=32 B heapSize=360 B 2024-11-23T22:49:19,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/default/TestHBaseWalOnEC/c0f4f302e3b6b1e7ce6f6aaf333f0c7e/.tmp/cf/1310b4cd00f141c38388233df444c120 is 36, key is row/cf:cq/1732402159484/Put/seqid=0 2024-11-23T22:49:19,733 WARN [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:19,733 WARN [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:19,738 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-232098387_22 at /127.0.0.1:59820 [Receiving block BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775616_1028] {}] datanode.DataXceiver(331): 127.0.0.1:35543:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59820 dst: /127.0.0.1:35543 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:19,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35543 is added to blk_-9223372036854775616_1029 (size=4787) 2024-11-23T22:49:19,743 WARN [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-23T22:49:19,743 INFO [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/default/TestHBaseWalOnEC/c0f4f302e3b6b1e7ce6f6aaf333f0c7e/.tmp/cf/1310b4cd00f141c38388233df444c120 2024-11-23T22:49:19,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/default/TestHBaseWalOnEC/c0f4f302e3b6b1e7ce6f6aaf333f0c7e/.tmp/cf/1310b4cd00f141c38388233df444c120 as hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/default/TestHBaseWalOnEC/c0f4f302e3b6b1e7ce6f6aaf333f0c7e/cf/1310b4cd00f141c38388233df444c120 2024-11-23T22:49:19,792 INFO [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/default/TestHBaseWalOnEC/c0f4f302e3b6b1e7ce6f6aaf333f0c7e/cf/1310b4cd00f141c38388233df444c120, entries=1, sequenceid=5, filesize=4.7 K 2024-11-23T22:49:19,799 INFO [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~32 B/32, heapSize ~344 B/344, currentSize=0 B/0 for c0f4f302e3b6b1e7ce6f6aaf333f0c7e in 125ms, sequenceid=5, compaction requested=false 2024-11-23T22:49:19,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestHBaseWalOnEC' 2024-11-23T22:49:19,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for c0f4f302e3b6b1e7ce6f6aaf333f0c7e: 2024-11-23T22:49:19,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e. 2024-11-23T22:49:19,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-23T22:49:19,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-23T22:49:19,812 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T22:49:19,812 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 305 msec 2024-11-23T22:49:19,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T22:49:19,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC in 319 msec 2024-11-23T22:49:19,816 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestHBaseWalOnEC completed 2024-11-23T22:49:19,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T22:49:19,829 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T22:49:19,829 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.TestHBaseWalOnEC.tearDown(TestHBaseWalOnEC.java:101) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T22:49:19,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:19,834 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:19,834 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T22:49:19,834 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T22:49:19,834 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1075752841, stopped=false 2024-11-23T22:49:19,834 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=dd96e8dc8369,32885,1732402151433 2024-11-23T22:49:19,900 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T22:49:19,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T22:49:19,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T22:49:19,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T22:49:19,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:19,900 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:19,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:19,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:19,900 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T22:49:19,901 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T22:49:19,901 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.TestHBaseWalOnEC.tearDown(TestHBaseWalOnEC.java:101) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T22:49:19,901 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:49:19,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:19,901 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:49:19,902 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:49:19,902 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:49:19,903 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd96e8dc8369,38153,1732402152392' ***** 2024-11-23T22:49:19,903 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T22:49:19,903 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd96e8dc8369,35195,1732402152524' ***** 2024-11-23T22:49:19,903 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T22:49:19,903 INFO [RS:1;dd96e8dc8369:35195 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T22:49:19,904 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T22:49:19,904 INFO [RS:0;dd96e8dc8369:38153 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T22:49:19,904 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd96e8dc8369,45809,1732402152586' ***** 2024-11-23T22:49:19,904 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T22:49:19,904 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T22:49:19,904 INFO [RS:0;dd96e8dc8369:38153 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T22:49:19,904 INFO [RS:1;dd96e8dc8369:35195 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T22:49:19,905 INFO [RS:1;dd96e8dc8369:35195 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T22:49:19,905 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.HRegionServer(3091): Received CLOSE for c0f4f302e3b6b1e7ce6f6aaf333f0c7e 2024-11-23T22:49:19,905 INFO [RS:1;dd96e8dc8369:35195 {}] regionserver.HRegionServer(959): stopping server dd96e8dc8369,35195,1732402152524 2024-11-23T22:49:19,905 INFO [RS:1;dd96e8dc8369:35195 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T22:49:19,905 INFO [RS:1;dd96e8dc8369:35195 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;dd96e8dc8369:35195. 2024-11-23T22:49:19,905 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T22:49:19,905 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.HRegionServer(959): stopping server dd96e8dc8369,38153,1732402152392 2024-11-23T22:49:19,905 DEBUG [RS:1;dd96e8dc8369:35195 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T22:49:19,905 INFO [RS:0;dd96e8dc8369:38153 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T22:49:19,905 DEBUG [RS:1;dd96e8dc8369:35195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:19,905 INFO [RS:0;dd96e8dc8369:38153 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;dd96e8dc8369:38153. 2024-11-23T22:49:19,905 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T22:49:19,906 DEBUG [RS:0;dd96e8dc8369:38153 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T22:49:19,906 INFO [RS:2;dd96e8dc8369:45809 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T22:49:19,906 DEBUG [RS:0;dd96e8dc8369:38153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:19,906 INFO [RS:2;dd96e8dc8369:45809 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T22:49:19,906 INFO [RS:1;dd96e8dc8369:35195 {}] regionserver.HRegionServer(976): stopping server dd96e8dc8369,35195,1732402152524; all regions closed. 2024-11-23T22:49:19,906 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer(959): stopping server dd96e8dc8369,45809,1732402152586 2024-11-23T22:49:19,906 INFO [RS:2;dd96e8dc8369:45809 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T22:49:19,906 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-23T22:49:19,906 INFO [RS:2;dd96e8dc8369:45809 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;dd96e8dc8369:45809. 2024-11-23T22:49:19,906 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c0f4f302e3b6b1e7ce6f6aaf333f0c7e, disabling compactions & flushes 2024-11-23T22:49:19,906 DEBUG [RS:0;dd96e8dc8369:38153 {}] regionserver.HRegionServer(1325): Online Regions={c0f4f302e3b6b1e7ce6f6aaf333f0c7e=TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e.} 2024-11-23T22:49:19,906 DEBUG [RS:2;dd96e8dc8369:45809 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T22:49:19,906 INFO [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e. 2024-11-23T22:49:19,906 DEBUG [RS:2;dd96e8dc8369:45809 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:19,906 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T22:49:19,906 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e. 2024-11-23T22:49:19,906 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e. after waiting 0 ms 2024-11-23T22:49:19,907 DEBUG [RS:0;dd96e8dc8369:38153 {}] regionserver.HRegionServer(1351): Waiting on c0f4f302e3b6b1e7ce6f6aaf333f0c7e 2024-11-23T22:49:19,907 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T22:49:19,907 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e. 2024-11-23T22:49:19,907 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T22:49:19,907 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T22:49:19,907 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T22:49:19,907 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-23T22:49:19,907 DEBUG [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-23T22:49:19,907 DEBUG [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T22:49:19,907 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T22:49:19,908 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T22:49:19,908 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T22:49:19,908 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T22:49:19,908 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T22:49:19,908 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.34 KB heapSize=3.38 KB 2024-11-23T22:49:19,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43763 is added to blk_1073741828_1016 (size=93) 2024-11-23T22:49:19,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35543 is added to blk_1073741828_1016 (size=93) 2024-11-23T22:49:19,920 DEBUG [RS:1;dd96e8dc8369:35195 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/oldWALs 2024-11-23T22:49:19,920 INFO [RS:1;dd96e8dc8369:35195 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dd96e8dc8369%2C35195%2C1732402152524:(num 1732402155105) 2024-11-23T22:49:19,920 DEBUG [RS:1;dd96e8dc8369:35195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:19,920 INFO [RS:1;dd96e8dc8369:35195 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:49:19,920 INFO [RS:1;dd96e8dc8369:35195 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T22:49:19,920 INFO [RS:1;dd96e8dc8369:35195 {}] hbase.ChoreService(370): Chore service for: regionserver/dd96e8dc8369:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T22:49:19,921 INFO [RS:1;dd96e8dc8369:35195 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T22:49:19,921 INFO [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T22:49:19,921 INFO [RS:1;dd96e8dc8369:35195 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T22:49:19,921 INFO [RS:1;dd96e8dc8369:35195 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T22:49:19,921 INFO [RS:1;dd96e8dc8369:35195 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T22:49:19,921 INFO [RS:1;dd96e8dc8369:35195 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35195 2024-11-23T22:49:19,924 INFO [regionserver/dd96e8dc8369:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-23T22:49:19,924 INFO [regionserver/dd96e8dc8369:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-23T22:49:19,928 INFO [regionserver/dd96e8dc8369:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-23T22:49:19,928 INFO [regionserver/dd96e8dc8369:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-23T22:49:19,932 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/default/TestHBaseWalOnEC/c0f4f302e3b6b1e7ce6f6aaf333f0c7e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-23T22:49:19,935 INFO [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e. 2024-11-23T22:49:19,935 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c0f4f302e3b6b1e7ce6f6aaf333f0c7e: Waiting for close lock at 1732402159906Running coprocessor pre-close hooks at 1732402159906Disabling compacts and flushes for region at 1732402159906Disabling writes for close at 1732402159907 (+1 ms)Writing region close event to WAL at 1732402159908 (+1 ms)Running coprocessor post-close hooks at 1732402159933 (+25 ms)Closed at 1732402159935 (+2 ms) 2024-11-23T22:49:19,935 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e. 2024-11-23T22:49:19,936 INFO [regionserver/dd96e8dc8369:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:49:19,941 INFO [regionserver/dd96e8dc8369:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:49:19,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T22:49:19,943 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd96e8dc8369,35195,1732402152524 2024-11-23T22:49:19,943 INFO [RS:1;dd96e8dc8369:35195 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T22:49:19,944 INFO [regionserver/dd96e8dc8369:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:49:19,944 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd96e8dc8369,35195,1732402152524] 2024-11-23T22:49:19,952 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740/.tmp/info/a71fc33a65cd41cdb7ec8bf67f0060e3 is 153, key is TestHBaseWalOnEC,,1732402158797.c0f4f302e3b6b1e7ce6f6aaf333f0c7e./info:regioninfo/1732402159278/Put/seqid=0 2024-11-23T22:49:19,955 WARN [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:19,955 WARN [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:19,960 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1052600213_22 at /127.0.0.1:58980 [Receiving block BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775600_1030] {}] datanode.DataXceiver(331): 127.0.0.1:36219:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58980 dst: /127.0.0.1:36219 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:19,964 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd96e8dc8369,35195,1732402152524 already deleted, retry=false 2024-11-23T22:49:19,964 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd96e8dc8369,35195,1732402152524 expired; onlineServers=2 2024-11-23T22:49:19,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36219 is added to blk_-9223372036854775600_1031 (size=6637) 2024-11-23T22:49:19,966 WARN [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-23T22:49:19,966 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.18 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740/.tmp/info/a71fc33a65cd41cdb7ec8bf67f0060e3 2024-11-23T22:49:19,998 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740/.tmp/ns/134c3ca270ff477e883182165b7a6610 is 43, key is default/ns:d/1732402158517/Put/seqid=0 2024-11-23T22:49:20,001 WARN [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:20,001 WARN [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:20,004 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1052600213_22 at /127.0.0.1:35144 [Receiving block BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775584_1032] {}] datanode.DataXceiver(331): 127.0.0.1:43763:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35144 dst: /127.0.0.1:43763 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:20,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43763 is added to blk_-9223372036854775584_1033 (size=5153) 2024-11-23T22:49:20,008 WARN [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-23T22:49:20,008 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740/.tmp/ns/134c3ca270ff477e883182165b7a6610 2024-11-23T22:49:20,036 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740/.tmp/table/a746e2faec4c408d86a6b77176ccd15a is 52, key is TestHBaseWalOnEC/table:state/1732402159292/Put/seqid=0 2024-11-23T22:49:20,038 WARN [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:20,038 WARN [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:20,041 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1052600213_22 at /127.0.0.1:35172 [Receiving block BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775568_1034] {}] datanode.DataXceiver(331): 127.0.0.1:43763:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35172 dst: /127.0.0.1:43763 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:20,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43763 is added to blk_-9223372036854775568_1035 (size=5249) 2024-11-23T22:49:20,049 WARN [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-23T22:49:20,049 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=96 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740/.tmp/table/a746e2faec4c408d86a6b77176ccd15a 2024-11-23T22:49:20,054 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:49:20,054 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35195-0x10169e2f9690002, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:49:20,054 INFO [RS:1;dd96e8dc8369:35195 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T22:49:20,054 INFO [RS:1;dd96e8dc8369:35195 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd96e8dc8369,35195,1732402152524; zookeeper connection closed. 2024-11-23T22:49:20,055 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@e42fb0f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@e42fb0f 2024-11-23T22:49:20,062 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740/.tmp/info/a71fc33a65cd41cdb7ec8bf67f0060e3 as hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740/info/a71fc33a65cd41cdb7ec8bf67f0060e3 2024-11-23T22:49:20,073 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740/info/a71fc33a65cd41cdb7ec8bf67f0060e3, entries=10, sequenceid=11, filesize=6.5 K 2024-11-23T22:49:20,075 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740/.tmp/ns/134c3ca270ff477e883182165b7a6610 as hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740/ns/134c3ca270ff477e883182165b7a6610 2024-11-23T22:49:20,086 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740/ns/134c3ca270ff477e883182165b7a6610, entries=2, sequenceid=11, filesize=5.0 K 2024-11-23T22:49:20,088 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740/.tmp/table/a746e2faec4c408d86a6b77176ccd15a as hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740/table/a746e2faec4c408d86a6b77176ccd15a 2024-11-23T22:49:20,098 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740/table/a746e2faec4c408d86a6b77176ccd15a, entries=2, sequenceid=11, filesize=5.1 K 2024-11-23T22:49:20,099 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.34 KB/1377, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 191ms, sequenceid=11, compaction requested=false 2024-11-23T22:49:20,099 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T22:49:20,107 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.HRegionServer(976): stopping server dd96e8dc8369,38153,1732402152392; all regions closed. 2024-11-23T22:49:20,107 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-23T22:49:20,108 DEBUG [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T22:49:20,108 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T22:49:20,108 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T22:49:20,108 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732402159907Running coprocessor pre-close hooks at 1732402159907Disabling compacts and flushes for region at 1732402159907Disabling writes for close at 1732402159908 (+1 ms)Obtaining lock to block concurrent updates at 1732402159908Preparing flush snapshotting stores in 1588230740 at 1732402159908Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1377, getHeapSize=3392, getOffHeapSize=0, getCellsCount=14 at 1732402159909 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732402159910 (+1 ms)Flushing 1588230740/info: creating writer at 1732402159911 (+1 ms)Flushing 1588230740/info: appending metadata at 1732402159946 (+35 ms)Flushing 1588230740/info: closing flushed file at 1732402159947 (+1 ms)Flushing 1588230740/ns: creating writer at 1732402159979 (+32 ms)Flushing 1588230740/ns: appending metadata at 1732402159996 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732402159996Flushing 1588230740/table: creating writer at 1732402160017 (+21 ms)Flushing 1588230740/table: appending metadata at 1732402160034 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732402160034Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cab9799: reopening flushed file at 1732402160060 (+26 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19c04535: reopening flushed file at 1732402160073 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1edc46ca: reopening flushed file at 1732402160086 (+13 ms)Finished flush of dataSize ~1.34 KB/1377, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 191ms, sequenceid=11, compaction requested=false at 1732402160099 (+13 ms)Writing region close event to WAL at 1732402160101 (+2 ms)Running coprocessor post-close hooks at 1732402160108 (+7 ms)Closed at 1732402160108 2024-11-23T22:49:20,109 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T22:49:20,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35543 is added to blk_1073741827_1015 (size=1298) 2024-11-23T22:49:20,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43763 is added to blk_1073741827_1015 (size=1298) 2024-11-23T22:49:20,112 DEBUG [RS:0;dd96e8dc8369:38153 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/oldWALs 2024-11-23T22:49:20,113 INFO [RS:0;dd96e8dc8369:38153 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dd96e8dc8369%2C38153%2C1732402152392:(num 1732402155112) 2024-11-23T22:49:20,113 DEBUG [RS:0;dd96e8dc8369:38153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:20,113 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:49:20,113 INFO [RS:0;dd96e8dc8369:38153 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T22:49:20,113 INFO [RS:0;dd96e8dc8369:38153 {}] hbase.ChoreService(370): Chore service for: regionserver/dd96e8dc8369:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T22:49:20,113 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T22:49:20,113 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T22:49:20,113 INFO [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T22:49:20,113 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T22:49:20,113 INFO [RS:0;dd96e8dc8369:38153 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T22:49:20,113 INFO [RS:0;dd96e8dc8369:38153 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38153 2024-11-23T22:49:20,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T22:49:20,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd96e8dc8369,38153,1732402152392 2024-11-23T22:49:20,122 INFO [RS:0;dd96e8dc8369:38153 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T22:49:20,133 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd96e8dc8369,38153,1732402152392] 2024-11-23T22:49:20,143 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd96e8dc8369,38153,1732402152392 already deleted, retry=false 2024-11-23T22:49:20,143 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd96e8dc8369,38153,1732402152392 expired; onlineServers=1 2024-11-23T22:49:20,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:49:20,233 INFO [RS:0;dd96e8dc8369:38153 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T22:49:20,233 INFO [RS:0;dd96e8dc8369:38153 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd96e8dc8369,38153,1732402152392; zookeeper connection closed. 2024-11-23T22:49:20,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38153-0x10169e2f9690001, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:49:20,234 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@25659d25 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@25659d25 2024-11-23T22:49:20,308 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer(976): stopping server dd96e8dc8369,45809,1732402152586; all regions closed. 2024-11-23T22:49:20,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35543 is added to blk_1073741829_1023 (size=2751) 2024-11-23T22:49:20,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36219 is added to blk_1073741829_1023 (size=2751) 2024-11-23T22:49:20,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43763 is added to blk_1073741829_1023 (size=2751) 2024-11-23T22:49:20,322 DEBUG [RS:2;dd96e8dc8369:45809 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/oldWALs 2024-11-23T22:49:20,322 INFO [RS:2;dd96e8dc8369:45809 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dd96e8dc8369%2C45809%2C1732402152586.meta:.meta(num 1732402158298) 2024-11-23T22:49:20,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43763 is added to blk_1073741826_1014 (size=93) 2024-11-23T22:49:20,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35543 is added to blk_1073741826_1014 (size=93) 2024-11-23T22:49:20,325 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/WALs/dd96e8dc8369,45809,1732402152586/dd96e8dc8369%2C45809%2C1732402152586.1732402155107 not finished, retry = 0 2024-11-23T22:49:20,432 DEBUG [RS:2;dd96e8dc8369:45809 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/oldWALs 2024-11-23T22:49:20,432 INFO [RS:2;dd96e8dc8369:45809 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dd96e8dc8369%2C45809%2C1732402152586:(num 1732402155107) 2024-11-23T22:49:20,432 DEBUG [RS:2;dd96e8dc8369:45809 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:20,432 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:49:20,432 INFO [RS:2;dd96e8dc8369:45809 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T22:49:20,433 INFO [RS:2;dd96e8dc8369:45809 {}] hbase.ChoreService(370): Chore service for: regionserver/dd96e8dc8369:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T22:49:20,433 INFO [RS:2;dd96e8dc8369:45809 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T22:49:20,433 INFO [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T22:49:20,434 INFO [RS:2;dd96e8dc8369:45809 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45809 2024-11-23T22:49:20,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd96e8dc8369,45809,1732402152586 2024-11-23T22:49:20,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T22:49:20,448 INFO [RS:2;dd96e8dc8369:45809 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T22:49:20,459 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd96e8dc8369,45809,1732402152586] 2024-11-23T22:49:20,469 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd96e8dc8369,45809,1732402152586 already deleted, retry=false 2024-11-23T22:49:20,470 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd96e8dc8369,45809,1732402152586 expired; onlineServers=0 2024-11-23T22:49:20,470 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'dd96e8dc8369,32885,1732402151433' ***** 2024-11-23T22:49:20,470 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T22:49:20,470 INFO [M:0;dd96e8dc8369:32885 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T22:49:20,471 INFO [M:0;dd96e8dc8369:32885 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T22:49:20,471 DEBUG [M:0;dd96e8dc8369:32885 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T22:49:20,471 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T22:49:20,471 DEBUG [M:0;dd96e8dc8369:32885 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T22:49:20,471 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402154570 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402154570,5,FailOnTimeoutGroup] 2024-11-23T22:49:20,471 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402154576 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402154576,5,FailOnTimeoutGroup] 2024-11-23T22:49:20,472 INFO [M:0;dd96e8dc8369:32885 {}] hbase.ChoreService(370): Chore service for: master/dd96e8dc8369:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T22:49:20,473 INFO [M:0;dd96e8dc8369:32885 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T22:49:20,473 DEBUG [M:0;dd96e8dc8369:32885 {}] master.HMaster(1795): Stopping service threads 2024-11-23T22:49:20,473 INFO [M:0;dd96e8dc8369:32885 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T22:49:20,473 INFO [M:0;dd96e8dc8369:32885 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T22:49:20,475 INFO [M:0;dd96e8dc8369:32885 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T22:49:20,475 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T22:49:20,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T22:49:20,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:20,480 DEBUG [M:0;dd96e8dc8369:32885 {}] zookeeper.ZKUtil(347): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T22:49:20,480 WARN [M:0;dd96e8dc8369:32885 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T22:49:20,481 INFO [M:0;dd96e8dc8369:32885 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/.lastflushedseqids 2024-11-23T22:49:20,490 WARN [M:0;dd96e8dc8369:32885 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:20,490 WARN [M:0;dd96e8dc8369:32885 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:20,493 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_801357093_22 at /127.0.0.1:59012 [Receiving block BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775552_1036] {}] datanode.DataXceiver(331): 127.0.0.1:36219:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59012 dst: /127.0.0.1:36219 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:20,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36219 is added to blk_-9223372036854775552_1037 (size=127) 2024-11-23T22:49:20,496 WARN [M:0;dd96e8dc8369:32885 {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-23T22:49:20,497 INFO [M:0;dd96e8dc8369:32885 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T22:49:20,497 INFO [M:0;dd96e8dc8369:32885 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T22:49:20,497 DEBUG [M:0;dd96e8dc8369:32885 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T22:49:20,497 INFO [M:0;dd96e8dc8369:32885 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:49:20,497 DEBUG [M:0;dd96e8dc8369:32885 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:49:20,497 DEBUG [M:0;dd96e8dc8369:32885 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T22:49:20,497 DEBUG [M:0;dd96e8dc8369:32885 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:49:20,497 INFO [M:0;dd96e8dc8369:32885 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=27.75 KB heapSize=35.34 KB 2024-11-23T22:49:20,516 DEBUG [M:0;dd96e8dc8369:32885 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e397a712a85c41f184b50bd3772c1510 is 82, key is hbase:meta,,1/info:regioninfo/1732402158399/Put/seqid=0 2024-11-23T22:49:20,518 WARN [M:0;dd96e8dc8369:32885 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:20,518 WARN [M:0;dd96e8dc8369:32885 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:20,521 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_801357093_22 at /127.0.0.1:35186 [Receiving block BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775536_1038] {}] datanode.DataXceiver(331): 127.0.0.1:43763:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35186 dst: /127.0.0.1:43763 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:20,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43763 is added to blk_-9223372036854775536_1039 (size=5672) 2024-11-23T22:49:20,525 WARN [M:0;dd96e8dc8369:32885 {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-23T22:49:20,525 INFO [M:0;dd96e8dc8369:32885 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e397a712a85c41f184b50bd3772c1510 2024-11-23T22:49:20,548 DEBUG [M:0;dd96e8dc8369:32885 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/80ccfd9296984bb4a05782db81c7165e is 748, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732402159299/Put/seqid=0 2024-11-23T22:49:20,551 WARN [M:0;dd96e8dc8369:32885 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:20,551 WARN [M:0;dd96e8dc8369:32885 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:20,553 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_801357093_22 at /127.0.0.1:35212 [Receiving block BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775520_1040] {}] datanode.DataXceiver(331): 127.0.0.1:43763:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35212 dst: /127.0.0.1:43763 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:20,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43763 is added to blk_-9223372036854775520_1041 (size=6442) 2024-11-23T22:49:20,558 WARN [M:0;dd96e8dc8369:32885 {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-23T22:49:20,558 INFO [M:0;dd96e8dc8369:32885 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.07 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/80ccfd9296984bb4a05782db81c7165e 2024-11-23T22:49:20,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:49:20,559 INFO [RS:2;dd96e8dc8369:45809 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T22:49:20,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45809-0x10169e2f9690003, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:49:20,559 INFO [RS:2;dd96e8dc8369:45809 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd96e8dc8369,45809,1732402152586; zookeeper connection closed. 2024-11-23T22:49:20,559 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@79af9f68 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@79af9f68 2024-11-23T22:49:20,560 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-23T22:49:20,584 DEBUG [M:0;dd96e8dc8369:32885 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/df379c91f57d4ba0bd3d77212c38a886 is 69, key is dd96e8dc8369,35195,1732402152524/rs:state/1732402154686/Put/seqid=0 2024-11-23T22:49:20,586 WARN [M:0;dd96e8dc8369:32885 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:20,586 WARN [M:0;dd96e8dc8369:32885 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-23T22:49:20,588 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_801357093_22 at /127.0.0.1:35234 [Receiving block BP-868742450-172.17.0.2-1732402146260:blk_-9223372036854775504_1042] {}] datanode.DataXceiver(331): 127.0.0.1:43763:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35234 dst: /127.0.0.1:43763 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:49:20,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43763 is added to blk_-9223372036854775504_1043 (size=5294) 2024-11-23T22:49:20,592 WARN [M:0;dd96e8dc8369:32885 {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-23T22:49:20,592 INFO [M:0;dd96e8dc8369:32885 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=195 B at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/df379c91f57d4ba0bd3d77212c38a886 2024-11-23T22:49:20,602 DEBUG [M:0;dd96e8dc8369:32885 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e397a712a85c41f184b50bd3772c1510 as hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e397a712a85c41f184b50bd3772c1510 2024-11-23T22:49:20,611 INFO [M:0;dd96e8dc8369:32885 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e397a712a85c41f184b50bd3772c1510, entries=8, sequenceid=76, filesize=5.5 K 2024-11-23T22:49:20,613 DEBUG [M:0;dd96e8dc8369:32885 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/80ccfd9296984bb4a05782db81c7165e as hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/80ccfd9296984bb4a05782db81c7165e 2024-11-23T22:49:20,621 INFO [M:0;dd96e8dc8369:32885 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/80ccfd9296984bb4a05782db81c7165e, entries=8, sequenceid=76, filesize=6.3 K 2024-11-23T22:49:20,623 DEBUG [M:0;dd96e8dc8369:32885 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/df379c91f57d4ba0bd3d77212c38a886 as hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/df379c91f57d4ba0bd3d77212c38a886 2024-11-23T22:49:20,631 INFO [M:0;dd96e8dc8369:32885 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/df379c91f57d4ba0bd3d77212c38a886, entries=3, sequenceid=76, filesize=5.2 K 2024-11-23T22:49:20,633 INFO [M:0;dd96e8dc8369:32885 {}] regionserver.HRegion(3140): Finished flush of dataSize ~27.75 KB/28414, heapSize ~35.04 KB/35880, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=76, compaction requested=false 2024-11-23T22:49:20,634 INFO [M:0;dd96e8dc8369:32885 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:49:20,634 DEBUG [M:0;dd96e8dc8369:32885 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732402160497Disabling compacts and flushes for region at 1732402160497Disabling writes for close at 1732402160497Obtaining lock to block concurrent updates at 1732402160497Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732402160497Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=28414, getHeapSize=36120, getOffHeapSize=0, getCellsCount=89 at 1732402160498 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732402160499 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732402160499Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732402160515 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732402160515Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732402160532 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732402160548 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732402160548Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732402160567 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732402160583 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732402160583Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@144ce830: reopening flushed file at 1732402160600 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b658595: reopening flushed file at 1732402160611 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6807db6: reopening flushed file at 1732402160622 (+11 ms)Finished flush of dataSize ~27.75 KB/28414, heapSize ~35.04 KB/35880, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=76, compaction requested=false at 1732402160633 (+11 ms)Writing region close event to WAL at 1732402160634 (+1 ms)Closed at 1732402160634 2024-11-23T22:49:20,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43763 is added to blk_1073741825_1011 (size=33873) 2024-11-23T22:49:20,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35543 is added to blk_1073741825_1011 (size=33873) 2024-11-23T22:49:20,639 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T22:49:20,639 INFO [M:0;dd96e8dc8369:32885 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T22:49:20,639 INFO [M:0;dd96e8dc8369:32885 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32885 2024-11-23T22:49:20,639 INFO [M:0;dd96e8dc8369:32885 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T22:49:20,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:49:20,748 INFO [M:0;dd96e8dc8369:32885 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T22:49:20,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x10169e2f9690000, quorum=127.0.0.1:57490, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:49:20,755 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2a9ecb50{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T22:49:20,758 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4067fd8a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T22:49:20,758 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T22:49:20,758 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@146c020c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T22:49:20,758 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@233bb3ea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/hadoop.log.dir/,STOPPED} 2024-11-23T22:49:20,761 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T22:49:20,761 WARN [BP-868742450-172.17.0.2-1732402146260 heartbeating to localhost/127.0.0.1:40647 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T22:49:20,761 WARN [BP-868742450-172.17.0.2-1732402146260 heartbeating to localhost/127.0.0.1:40647 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-868742450-172.17.0.2-1732402146260 (Datanode Uuid abef8c5b-3c1a-4737-8cde-1b8f158c6a26) service to localhost/127.0.0.1:40647 2024-11-23T22:49:20,761 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T22:49:20,763 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/cluster_10dec550-ad07-9099-3875-13a2f1d6fb1f/data/data5/current/BP-868742450-172.17.0.2-1732402146260 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T22:49:20,763 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/cluster_10dec550-ad07-9099-3875-13a2f1d6fb1f/data/data6/current/BP-868742450-172.17.0.2-1732402146260 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T22:49:20,764 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T22:49:20,766 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@14402056{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T22:49:20,766 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3e1cb3ec{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T22:49:20,766 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T22:49:20,766 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@65cd6e19{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T22:49:20,766 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5435fd88{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/hadoop.log.dir/,STOPPED} 2024-11-23T22:49:20,768 WARN [BP-868742450-172.17.0.2-1732402146260 heartbeating to localhost/127.0.0.1:40647 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T22:49:20,768 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T22:49:20,769 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T22:49:20,769 WARN [BP-868742450-172.17.0.2-1732402146260 heartbeating to localhost/127.0.0.1:40647 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-868742450-172.17.0.2-1732402146260 (Datanode Uuid 6222555d-d845-4165-87b4-a4b99c8e8037) service to localhost/127.0.0.1:40647 2024-11-23T22:49:20,769 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/cluster_10dec550-ad07-9099-3875-13a2f1d6fb1f/data/data3/current/BP-868742450-172.17.0.2-1732402146260 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T22:49:20,770 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/cluster_10dec550-ad07-9099-3875-13a2f1d6fb1f/data/data4/current/BP-868742450-172.17.0.2-1732402146260 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T22:49:20,770 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T22:49:20,773 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5b23cf15{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T22:49:20,773 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7f9e5902{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T22:49:20,773 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T22:49:20,773 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3b3a0659{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T22:49:20,773 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3665148e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/hadoop.log.dir/,STOPPED} 2024-11-23T22:49:20,775 WARN [BP-868742450-172.17.0.2-1732402146260 heartbeating to localhost/127.0.0.1:40647 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T22:49:20,775 WARN [BP-868742450-172.17.0.2-1732402146260 heartbeating to localhost/127.0.0.1:40647 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-868742450-172.17.0.2-1732402146260 (Datanode Uuid ff009774-bb68-4f04-a785-f72b51c43e68) service to localhost/127.0.0.1:40647 2024-11-23T22:49:20,776 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/cluster_10dec550-ad07-9099-3875-13a2f1d6fb1f/data/data1/current/BP-868742450-172.17.0.2-1732402146260 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T22:49:20,776 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/cluster_10dec550-ad07-9099-3875-13a2f1d6fb1f/data/data2/current/BP-868742450-172.17.0.2-1732402146260 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T22:49:20,776 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T22:49:20,777 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T22:49:20,777 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T22:49:20,787 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@62d6efd9{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T22:49:20,788 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d35a1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T22:49:20,788 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T22:49:20,788 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ce709a8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T22:49:20,788 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@760c69c0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/hadoop.log.dir/,STOPPED} 2024-11-23T22:49:20,796 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T22:49:20,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-23T22:49:20,831 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestHBaseWalOnEC#testReadWrite[0] Thread=87 (was 157), OpenFileDescriptor=433 (was 391) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=558 (was 587), ProcessCount=11 (was 11), AvailableMemoryMB=6916 (was 7211) 2024-11-23T22:49:20,836 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestHBaseWalOnEC#testReadWrite[1] Thread=87, OpenFileDescriptor=433, MaxFileDescriptor=1048576, SystemLoadAverage=558, ProcessCount=11, AvailableMemoryMB=6916 2024-11-23T22:49:20,836 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-23T22:49:20,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/hadoop.log.dir so I do NOT create it in target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c 2024-11-23T22:49:20,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fe75707-3634-d754-ddad-4195ce7437c6/hadoop.tmp.dir so I do NOT create it in target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c 2024-11-23T22:49:20,837 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/cluster_3cba1636-64cb-f804-3a33-58a581002c60, deleteOnExit=true 2024-11-23T22:49:20,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-23T22:49:20,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/test.cache.data in system properties and HBase conf 2024-11-23T22:49:20,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/hadoop.tmp.dir in system properties and HBase conf 2024-11-23T22:49:20,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/hadoop.log.dir in system properties and HBase conf 2024-11-23T22:49:20,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-23T22:49:20,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-23T22:49:20,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-23T22:49:20,838 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:49:20,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-23T22:49:20,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-23T22:49:20,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-23T22:49:20,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T22:49:20,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-23T22:49:20,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-23T22:49:20,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T22:49:20,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T22:49:20,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-23T22:49:20,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/nfs.dump.dir in system properties and HBase conf 2024-11-23T22:49:20,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/java.io.tmpdir in system properties and HBase conf 2024-11-23T22:49:20,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T22:49:20,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-23T22:49:20,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-23T22:49:21,195 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T22:49:21,200 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T22:49:21,201 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T22:49:21,201 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T22:49:21,202 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T22:49:21,202 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T22:49:21,203 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@17794d45{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/hadoop.log.dir/,AVAILABLE} 2024-11-23T22:49:21,203 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4072566{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T22:49:21,298 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@10db3f46{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/java.io.tmpdir/jetty-localhost-37131-hadoop-hdfs-3_4_1-tests_jar-_-any-7504040047169488778/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T22:49:21,299 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43a7f4cb{HTTP/1.1, (http/1.1)}{localhost:37131} 2024-11-23T22:49:21,299 INFO [Time-limited test {}] server.Server(415): Started @17781ms 2024-11-23T22:49:21,563 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T22:49:21,568 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T22:49:21,569 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T22:49:21,569 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T22:49:21,569 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T22:49:21,569 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5e23bf16{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/hadoop.log.dir/,AVAILABLE} 2024-11-23T22:49:21,570 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@68e19264{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T22:49:21,662 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@11cbed31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/java.io.tmpdir/jetty-localhost-38209-hadoop-hdfs-3_4_1-tests_jar-_-any-3207794992735144356/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T22:49:21,663 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11812ea4{HTTP/1.1, (http/1.1)}{localhost:38209} 2024-11-23T22:49:21,663 INFO [Time-limited test {}] server.Server(415): Started @18145ms 2024-11-23T22:49:21,664 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T22:49:21,694 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T22:49:21,698 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T22:49:21,699 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T22:49:21,699 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T22:49:21,699 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T22:49:21,700 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5c773fd1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/hadoop.log.dir/,AVAILABLE} 2024-11-23T22:49:21,701 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72785dee{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T22:49:21,795 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@25cc5f4d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/java.io.tmpdir/jetty-localhost-35613-hadoop-hdfs-3_4_1-tests_jar-_-any-5996133391866268677/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T22:49:21,795 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6686fe53{HTTP/1.1, (http/1.1)}{localhost:35613} 2024-11-23T22:49:21,795 INFO [Time-limited test {}] server.Server(415): Started @18277ms 2024-11-23T22:49:21,797 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T22:49:21,826 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T22:49:21,831 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T22:49:21,831 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T22:49:21,831 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T22:49:21,831 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T22:49:21,832 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e74bc8b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/hadoop.log.dir/,AVAILABLE} 2024-11-23T22:49:21,832 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3c97cc8b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T22:49:21,928 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7c3d82b3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/java.io.tmpdir/jetty-localhost-46633-hadoop-hdfs-3_4_1-tests_jar-_-any-8670291930025022385/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T22:49:21,929 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@190023f0{HTTP/1.1, (http/1.1)}{localhost:46633} 2024-11-23T22:49:21,929 INFO [Time-limited test {}] server.Server(415): Started @18411ms 2024-11-23T22:49:21,930 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T22:49:23,152 WARN [Thread-624 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/cluster_3cba1636-64cb-f804-3a33-58a581002c60/data/data2/current/BP-710832709-172.17.0.2-1732402160863/current, will proceed with Du for space computation calculation, 2024-11-23T22:49:23,152 WARN [Thread-623 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/cluster_3cba1636-64cb-f804-3a33-58a581002c60/data/data1/current/BP-710832709-172.17.0.2-1732402160863/current, will proceed with Du for space computation calculation, 2024-11-23T22:49:23,173 WARN [Thread-563 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T22:49:23,176 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7a861b91cc3b6de6 with lease ID 0x5a4cda4343fef698: Processing first storage report for DS-dcb1b87b-5124-48c0-8911-283c2bfdbb52 from datanode DatanodeRegistration(127.0.0.1:41827, datanodeUuid=d5ccd38d-3b80-4f7f-930e-4ad9e622e6cc, infoPort=36221, infoSecurePort=0, ipcPort=37085, storageInfo=lv=-57;cid=testClusterID;nsid=2057744404;c=1732402160863) 2024-11-23T22:49:23,176 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7a861b91cc3b6de6 with lease ID 0x5a4cda4343fef698: from storage DS-dcb1b87b-5124-48c0-8911-283c2bfdbb52 node DatanodeRegistration(127.0.0.1:41827, datanodeUuid=d5ccd38d-3b80-4f7f-930e-4ad9e622e6cc, infoPort=36221, infoSecurePort=0, ipcPort=37085, storageInfo=lv=-57;cid=testClusterID;nsid=2057744404;c=1732402160863), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-23T22:49:23,176 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7a861b91cc3b6de6 with lease ID 0x5a4cda4343fef698: Processing first storage report for DS-b58494a1-cff9-4df7-8ac5-5328d810bb47 from datanode DatanodeRegistration(127.0.0.1:41827, datanodeUuid=d5ccd38d-3b80-4f7f-930e-4ad9e622e6cc, infoPort=36221, infoSecurePort=0, ipcPort=37085, storageInfo=lv=-57;cid=testClusterID;nsid=2057744404;c=1732402160863) 2024-11-23T22:49:23,176 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7a861b91cc3b6de6 with lease ID 0x5a4cda4343fef698: from storage DS-b58494a1-cff9-4df7-8ac5-5328d810bb47 node DatanodeRegistration(127.0.0.1:41827, datanodeUuid=d5ccd38d-3b80-4f7f-930e-4ad9e622e6cc, infoPort=36221, infoSecurePort=0, ipcPort=37085, storageInfo=lv=-57;cid=testClusterID;nsid=2057744404;c=1732402160863), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T22:49:23,348 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T22:49:23,348 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T22:49:23,349 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T22:49:23,420 WARN [Thread-634 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/cluster_3cba1636-64cb-f804-3a33-58a581002c60/data/data3/current/BP-710832709-172.17.0.2-1732402160863/current, will proceed with Du for space computation calculation, 2024-11-23T22:49:23,420 WARN [Thread-635 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/cluster_3cba1636-64cb-f804-3a33-58a581002c60/data/data4/current/BP-710832709-172.17.0.2-1732402160863/current, will proceed with Du for space computation calculation, 2024-11-23T22:49:23,440 WARN [Thread-586 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T22:49:23,443 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3259a87099b5e607 with lease ID 0x5a4cda4343fef699: Processing first storage report for DS-452a0f3c-9518-45ec-9b65-d08b8109f150 from datanode DatanodeRegistration(127.0.0.1:39807, datanodeUuid=7b537fac-16c8-4935-8cfd-e900380da330, infoPort=43843, infoSecurePort=0, ipcPort=35755, storageInfo=lv=-57;cid=testClusterID;nsid=2057744404;c=1732402160863) 2024-11-23T22:49:23,443 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3259a87099b5e607 with lease ID 0x5a4cda4343fef699: from storage DS-452a0f3c-9518-45ec-9b65-d08b8109f150 node DatanodeRegistration(127.0.0.1:39807, datanodeUuid=7b537fac-16c8-4935-8cfd-e900380da330, infoPort=43843, infoSecurePort=0, ipcPort=35755, storageInfo=lv=-57;cid=testClusterID;nsid=2057744404;c=1732402160863), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T22:49:23,443 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3259a87099b5e607 with lease ID 0x5a4cda4343fef699: Processing first storage report for DS-1c56a8f0-ab5f-4456-a6f8-39b751cb8d1c from datanode DatanodeRegistration(127.0.0.1:39807, datanodeUuid=7b537fac-16c8-4935-8cfd-e900380da330, infoPort=43843, infoSecurePort=0, ipcPort=35755, storageInfo=lv=-57;cid=testClusterID;nsid=2057744404;c=1732402160863) 2024-11-23T22:49:23,444 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3259a87099b5e607 with lease ID 0x5a4cda4343fef699: from storage DS-1c56a8f0-ab5f-4456-a6f8-39b751cb8d1c node DatanodeRegistration(127.0.0.1:39807, datanodeUuid=7b537fac-16c8-4935-8cfd-e900380da330, infoPort=43843, infoSecurePort=0, ipcPort=35755, storageInfo=lv=-57;cid=testClusterID;nsid=2057744404;c=1732402160863), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T22:49:23,461 WARN [Thread-645 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/cluster_3cba1636-64cb-f804-3a33-58a581002c60/data/data5/current/BP-710832709-172.17.0.2-1732402160863/current, will proceed with Du for space computation calculation, 2024-11-23T22:49:23,461 WARN [Thread-646 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/cluster_3cba1636-64cb-f804-3a33-58a581002c60/data/data6/current/BP-710832709-172.17.0.2-1732402160863/current, will proceed with Du for space computation calculation, 2024-11-23T22:49:23,484 WARN [Thread-608 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T22:49:23,487 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x82568142a19f1f4b with lease ID 0x5a4cda4343fef69a: Processing first storage report for DS-b1c65f39-79bc-40b0-800d-3dc88575d66b from datanode DatanodeRegistration(127.0.0.1:38213, datanodeUuid=4fafd9b6-edc3-47a5-8faa-c6465e36b7ec, infoPort=34501, infoSecurePort=0, ipcPort=40003, storageInfo=lv=-57;cid=testClusterID;nsid=2057744404;c=1732402160863) 2024-11-23T22:49:23,487 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x82568142a19f1f4b with lease ID 0x5a4cda4343fef69a: from storage DS-b1c65f39-79bc-40b0-800d-3dc88575d66b node DatanodeRegistration(127.0.0.1:38213, datanodeUuid=4fafd9b6-edc3-47a5-8faa-c6465e36b7ec, infoPort=34501, infoSecurePort=0, ipcPort=40003, storageInfo=lv=-57;cid=testClusterID;nsid=2057744404;c=1732402160863), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T22:49:23,487 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x82568142a19f1f4b with lease ID 0x5a4cda4343fef69a: Processing first storage report for DS-2c306fbd-181f-4348-884f-f80e148a1657 from datanode DatanodeRegistration(127.0.0.1:38213, datanodeUuid=4fafd9b6-edc3-47a5-8faa-c6465e36b7ec, infoPort=34501, infoSecurePort=0, ipcPort=40003, storageInfo=lv=-57;cid=testClusterID;nsid=2057744404;c=1732402160863) 2024-11-23T22:49:23,488 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x82568142a19f1f4b with lease ID 0x5a4cda4343fef69a: from storage DS-2c306fbd-181f-4348-884f-f80e148a1657 node DatanodeRegistration(127.0.0.1:38213, datanodeUuid=4fafd9b6-edc3-47a5-8faa-c6465e36b7ec, infoPort=34501, infoSecurePort=0, ipcPort=40003, storageInfo=lv=-57;cid=testClusterID;nsid=2057744404;c=1732402160863), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T22:49:23,581 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c 2024-11-23T22:49:23,584 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/cluster_3cba1636-64cb-f804-3a33-58a581002c60/zookeeper_0, clientPort=64210, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/cluster_3cba1636-64cb-f804-3a33-58a581002c60/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/cluster_3cba1636-64cb-f804-3a33-58a581002c60/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T22:49:23,585 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64210 2024-11-23T22:49:23,585 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:23,587 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:23,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41827 is added to blk_1073741825_1001 (size=7) 2024-11-23T22:49:23,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741825_1001 (size=7) 2024-11-23T22:49:23,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741825_1001 (size=7) 2024-11-23T22:49:23,602 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da with version=8 2024-11-23T22:49:23,602 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:40647/user/jenkins/test-data/ed05fbae-d7ae-2bbd-4f21-4c1af2f7b9ef/hbase-staging 2024-11-23T22:49:23,605 INFO [Time-limited test {}] client.ConnectionUtils(128): master/dd96e8dc8369:0 server-side Connection retries=45 2024-11-23T22:49:23,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:23,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:23,605 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T22:49:23,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:23,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T22:49:23,605 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T22:49:23,605 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T22:49:23,606 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36871 2024-11-23T22:49:23,608 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36871 connecting to ZooKeeper ensemble=127.0.0.1:64210 2024-11-23T22:49:23,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:368710x0, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T22:49:23,708 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36871-0x10169e32b2b0000 connected 2024-11-23T22:49:23,785 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:23,787 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:23,789 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:49:23,789 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da, hbase.cluster.distributed=false 2024-11-23T22:49:23,791 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T22:49:23,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36871 2024-11-23T22:49:23,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36871 2024-11-23T22:49:23,793 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36871 2024-11-23T22:49:23,794 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36871 2024-11-23T22:49:23,794 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36871 2024-11-23T22:49:23,807 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd96e8dc8369:0 server-side Connection retries=45 2024-11-23T22:49:23,807 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:23,808 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:23,808 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T22:49:23,808 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:23,808 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T22:49:23,808 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T22:49:23,808 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T22:49:23,808 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36415 2024-11-23T22:49:23,810 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36415 connecting to ZooKeeper ensemble=127.0.0.1:64210 2024-11-23T22:49:23,811 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:23,813 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:23,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:364150x0, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T22:49:23,828 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:364150x0, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:49:23,828 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36415-0x10169e32b2b0001 connected 2024-11-23T22:49:23,828 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T22:49:23,829 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T22:49:23,829 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T22:49:23,830 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T22:49:23,831 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36415 2024-11-23T22:49:23,831 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36415 2024-11-23T22:49:23,832 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36415 2024-11-23T22:49:23,835 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36415 2024-11-23T22:49:23,835 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36415 2024-11-23T22:49:23,849 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd96e8dc8369:0 server-side Connection retries=45 2024-11-23T22:49:23,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:23,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:23,849 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T22:49:23,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:23,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T22:49:23,849 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T22:49:23,849 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T22:49:23,850 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39195 2024-11-23T22:49:23,851 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39195 connecting to ZooKeeper ensemble=127.0.0.1:64210 2024-11-23T22:49:23,852 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:23,854 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:23,867 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-23T22:49:23,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:391950x0, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T22:49:23,870 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39195-0x10169e32b2b0002 connected 2024-11-23T22:49:23,870 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:49:23,870 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T22:49:23,871 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T22:49:23,872 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T22:49:23,873 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T22:49:23,875 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39195 2024-11-23T22:49:23,876 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39195 2024-11-23T22:49:23,876 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39195 2024-11-23T22:49:23,879 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39195 2024-11-23T22:49:23,888 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39195 2024-11-23T22:49:23,908 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T22:49:23,908 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-23T22:49:23,910 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd96e8dc8369:0 server-side Connection retries=45 2024-11-23T22:49:23,911 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:23,911 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:23,911 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T22:49:23,911 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:49:23,911 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T22:49:23,911 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T22:49:23,911 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T22:49:23,912 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45185 2024-11-23T22:49:23,915 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45185 connecting to ZooKeeper ensemble=127.0.0.1:64210 2024-11-23T22:49:23,916 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:23,918 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:23,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:451850x0, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T22:49:23,933 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:451850x0, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:49:23,933 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45185-0x10169e32b2b0003 connected 2024-11-23T22:49:23,933 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T22:49:23,934 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T22:49:23,934 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T22:49:23,936 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T22:49:23,937 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45185 2024-11-23T22:49:23,938 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45185 2024-11-23T22:49:23,940 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45185 2024-11-23T22:49:23,940 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45185 2024-11-23T22:49:23,941 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45185 2024-11-23T22:49:23,955 DEBUG [M:0;dd96e8dc8369:36871 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;dd96e8dc8369:36871 2024-11-23T22:49:23,956 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/dd96e8dc8369,36871,1732402163604 2024-11-23T22:49:23,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:49:23,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:49:23,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:49:23,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:49:23,964 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/dd96e8dc8369,36871,1732402163604 2024-11-23T22:49:23,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:23,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T22:49:23,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T22:49:23,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T22:49:23,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:23,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:23,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:23,975 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T22:49:23,976 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/dd96e8dc8369,36871,1732402163604 from backup master directory 2024-11-23T22:49:23,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:49:23,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:49:23,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/dd96e8dc8369,36871,1732402163604 2024-11-23T22:49:23,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:49:23,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:49:23,985 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T22:49:23,985 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=dd96e8dc8369,36871,1732402163604 2024-11-23T22:49:23,991 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/hbase.id] with ID: 98b63b6d-26a6-4031-a18a-9eb247a13068 2024-11-23T22:49:23,991 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/.tmp/hbase.id 2024-11-23T22:49:24,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741826_1002 (size=42) 2024-11-23T22:49:24,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41827 is added to blk_1073741826_1002 (size=42) 2024-11-23T22:49:24,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741826_1002 (size=42) 2024-11-23T22:49:24,001 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/.tmp/hbase.id]:[hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/hbase.id] 2024-11-23T22:49:24,016 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T22:49:24,016 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T22:49:24,018 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-23T22:49:24,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741827_1003 (size=196) 2024-11-23T22:49:24,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741827_1003 (size=196) 2024-11-23T22:49:24,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41827 is added to blk_1073741827_1003 (size=196) 2024-11-23T22:49:24,041 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T22:49:24,042 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T22:49:24,042 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-23T22:49:24,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741828_1004 (size=1189) 2024-11-23T22:49:24,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41827 is added to blk_1073741828_1004 (size=1189) 2024-11-23T22:49:24,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741828_1004 (size=1189) 2024-11-23T22:49:24,056 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store 2024-11-23T22:49:24,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41827 is added to blk_1073741829_1005 (size=34) 2024-11-23T22:49:24,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741829_1005 (size=34) 2024-11-23T22:49:24,065 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:49:24,065 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T22:49:24,065 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:49:24,065 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:49:24,065 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T22:49:24,065 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:49:24,065 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:49:24,065 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732402164065Disabling compacts and flushes for region at 1732402164065Disabling writes for close at 1732402164065Writing region close event to WAL at 1732402164065Closed at 1732402164065 2024-11-23T22:49:24,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741829_1005 (size=34) 2024-11-23T22:49:24,066 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store/.initializing 2024-11-23T22:49:24,066 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/WALs/dd96e8dc8369,36871,1732402163604 2024-11-23T22:49:24,070 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C36871%2C1732402163604, suffix=, logDir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/WALs/dd96e8dc8369,36871,1732402163604, archiveDir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/oldWALs, maxLogs=10 2024-11-23T22:49:24,070 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C36871%2C1732402163604.1732402164070 2024-11-23T22:49:24,080 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/WALs/dd96e8dc8369,36871,1732402163604/dd96e8dc8369%2C36871%2C1732402163604.1732402164070 2024-11-23T22:49:24,082 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36221:36221),(127.0.0.1/127.0.0.1:34501:34501),(127.0.0.1/127.0.0.1:43843:43843)] 2024-11-23T22:49:24,083 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T22:49:24,083 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:49:24,084 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:24,084 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:24,086 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:24,088 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T22:49:24,088 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:24,089 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:49:24,089 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:24,091 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T22:49:24,091 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:24,092 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T22:49:24,093 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:24,097 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T22:49:24,097 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:24,098 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T22:49:24,098 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:24,101 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T22:49:24,101 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:24,102 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T22:49:24,102 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:24,103 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:24,104 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:24,106 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:24,106 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:24,106 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T22:49:24,108 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:49:24,111 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T22:49:24,112 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75035402, jitterRate=0.11811462044715881}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T22:49:24,113 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732402164084Initializing all the Stores at 1732402164085 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402164086 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402164086Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402164086Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402164086Cleaning up temporary data from old regions at 1732402164106 (+20 ms)Region opened successfully at 1732402164113 (+7 ms) 2024-11-23T22:49:24,114 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T22:49:24,119 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@376b1c3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0 2024-11-23T22:49:24,120 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T22:49:24,120 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T22:49:24,120 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T22:49:24,120 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T22:49:24,121 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-23T22:49:24,121 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-23T22:49:24,121 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T22:49:24,124 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T22:49:24,125 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T22:49:24,132 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T22:49:24,133 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T22:49:24,134 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T22:49:24,143 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T22:49:24,143 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T22:49:24,144 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T22:49:24,153 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T22:49:24,154 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T22:49:24,163 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T22:49:24,165 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T22:49:24,174 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T22:49:24,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T22:49:24,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T22:49:24,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T22:49:24,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T22:49:24,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,186 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=dd96e8dc8369,36871,1732402163604, sessionid=0x10169e32b2b0000, setting cluster-up flag (Was=false) 2024-11-23T22:49:24,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,238 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T22:49:24,243 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,36871,1732402163604 2024-11-23T22:49:24,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,301 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T22:49:24,302 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,36871,1732402163604 2024-11-23T22:49:24,303 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T22:49:24,306 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T22:49:24,306 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T22:49:24,306 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T22:49:24,306 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: dd96e8dc8369,36871,1732402163604 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T22:49:24,308 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5 2024-11-23T22:49:24,308 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5 2024-11-23T22:49:24,308 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5 2024-11-23T22:49:24,308 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5 2024-11-23T22:49:24,308 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/dd96e8dc8369:0, corePoolSize=10, maxPoolSize=10 2024-11-23T22:49:24,308 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,308 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2 2024-11-23T22:49:24,309 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,309 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732402194309 2024-11-23T22:49:24,309 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T22:49:24,309 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T22:49:24,310 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T22:49:24,310 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T22:49:24,310 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T22:49:24,310 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T22:49:24,310 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,310 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T22:49:24,310 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T22:49:24,310 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T22:49:24,311 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T22:49:24,311 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T22:49:24,311 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T22:49:24,311 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T22:49:24,311 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402164311,5,FailOnTimeoutGroup] 2024-11-23T22:49:24,311 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402164311,5,FailOnTimeoutGroup] 2024-11-23T22:49:24,311 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,311 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T22:49:24,311 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,311 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,312 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:24,313 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T22:49:24,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41827 is added to blk_1073741831_1007 (size=1321) 2024-11-23T22:49:24,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741831_1007 (size=1321) 2024-11-23T22:49:24,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741831_1007 (size=1321) 2024-11-23T22:49:24,324 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T22:49:24,324 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da 2024-11-23T22:49:24,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741832_1008 (size=32) 2024-11-23T22:49:24,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741832_1008 (size=32) 2024-11-23T22:49:24,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41827 is added to blk_1073741832_1008 (size=32) 2024-11-23T22:49:24,334 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:49:24,335 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T22:49:24,337 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T22:49:24,337 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:24,337 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:49:24,337 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T22:49:24,339 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T22:49:24,339 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:24,339 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:49:24,339 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T22:49:24,341 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T22:49:24,341 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:24,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:49:24,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T22:49:24,343 INFO [RS:0;dd96e8dc8369:36415 {}] regionserver.HRegionServer(746): ClusterId : 98b63b6d-26a6-4031-a18a-9eb247a13068 2024-11-23T22:49:24,343 DEBUG [RS:0;dd96e8dc8369:36415 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T22:49:24,343 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.HRegionServer(746): ClusterId : 98b63b6d-26a6-4031-a18a-9eb247a13068 2024-11-23T22:49:24,343 DEBUG [RS:2;dd96e8dc8369:45185 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T22:49:24,343 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T22:49:24,343 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:24,344 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:49:24,344 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T22:49:24,345 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740 2024-11-23T22:49:24,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740 2024-11-23T22:49:24,347 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.HRegionServer(746): ClusterId : 98b63b6d-26a6-4031-a18a-9eb247a13068 2024-11-23T22:49:24,347 DEBUG [RS:1;dd96e8dc8369:39195 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T22:49:24,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T22:49:24,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T22:49:24,348 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T22:49:24,349 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T22:49:24,351 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T22:49:24,352 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61678370, jitterRate=-0.08092066645622253}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T22:49:24,353 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732402164334Initializing all the Stores at 1732402164335 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402164335Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402164335Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402164335Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402164335Cleaning up temporary data from old regions at 1732402164347 (+12 ms)Region opened successfully at 1732402164353 (+6 ms) 2024-11-23T22:49:24,353 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T22:49:24,353 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T22:49:24,353 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T22:49:24,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T22:49:24,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T22:49:24,356 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T22:49:24,356 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732402164353Disabling compacts and flushes for region at 1732402164353Disabling writes for close at 1732402164354 (+1 ms)Writing region close event to WAL at 1732402164356 (+2 ms)Closed at 1732402164356 2024-11-23T22:49:24,358 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T22:49:24,358 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T22:49:24,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T22:49:24,360 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T22:49:24,362 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T22:49:24,364 DEBUG [RS:0;dd96e8dc8369:36415 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T22:49:24,364 DEBUG [RS:0;dd96e8dc8369:36415 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T22:49:24,365 DEBUG [RS:1;dd96e8dc8369:39195 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T22:49:24,365 DEBUG [RS:1;dd96e8dc8369:39195 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T22:49:24,374 DEBUG [RS:2;dd96e8dc8369:45185 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T22:49:24,374 DEBUG [RS:2;dd96e8dc8369:45185 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T22:49:24,395 DEBUG [RS:0;dd96e8dc8369:36415 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T22:49:24,396 DEBUG [RS:1;dd96e8dc8369:39195 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T22:49:24,396 DEBUG [RS:0;dd96e8dc8369:36415 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598a85c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0 2024-11-23T22:49:24,396 DEBUG [RS:1;dd96e8dc8369:39195 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23f29fdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0 2024-11-23T22:49:24,396 DEBUG [RS:2;dd96e8dc8369:45185 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T22:49:24,396 DEBUG [RS:2;dd96e8dc8369:45185 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@252853c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0 2024-11-23T22:49:24,407 DEBUG [RS:2;dd96e8dc8369:45185 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;dd96e8dc8369:45185 2024-11-23T22:49:24,407 DEBUG [RS:1;dd96e8dc8369:39195 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;dd96e8dc8369:39195 2024-11-23T22:49:24,407 DEBUG [RS:0;dd96e8dc8369:36415 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;dd96e8dc8369:36415 2024-11-23T22:49:24,407 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T22:49:24,407 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T22:49:24,407 INFO [RS:0;dd96e8dc8369:36415 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T22:49:24,407 DEBUG [RS:2;dd96e8dc8369:45185 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T22:49:24,407 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T22:49:24,407 INFO [RS:0;dd96e8dc8369:36415 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T22:49:24,407 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T22:49:24,407 DEBUG [RS:0;dd96e8dc8369:36415 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T22:49:24,407 DEBUG [RS:1;dd96e8dc8369:39195 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T22:49:24,408 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,36871,1732402163604 with port=39195, startcode=1732402163849 2024-11-23T22:49:24,408 INFO [RS:0;dd96e8dc8369:36415 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,36871,1732402163604 with port=36415, startcode=1732402163807 2024-11-23T22:49:24,408 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,36871,1732402163604 with port=45185, startcode=1732402163910 2024-11-23T22:49:24,408 DEBUG [RS:1;dd96e8dc8369:39195 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T22:49:24,408 DEBUG [RS:2;dd96e8dc8369:45185 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T22:49:24,408 DEBUG [RS:0;dd96e8dc8369:36415 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T22:49:24,410 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43471, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T22:49:24,410 INFO [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41369, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T22:49:24,410 INFO [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42215, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T22:49:24,411 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36871 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd96e8dc8369,39195,1732402163849 2024-11-23T22:49:24,411 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36871 {}] master.ServerManager(517): Registering regionserver=dd96e8dc8369,39195,1732402163849 2024-11-23T22:49:24,413 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36871 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd96e8dc8369,36415,1732402163807 2024-11-23T22:49:24,413 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36871 {}] master.ServerManager(517): Registering regionserver=dd96e8dc8369,36415,1732402163807 2024-11-23T22:49:24,413 DEBUG [RS:1;dd96e8dc8369:39195 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da 2024-11-23T22:49:24,413 DEBUG [RS:1;dd96e8dc8369:39195 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42261 2024-11-23T22:49:24,413 DEBUG [RS:1;dd96e8dc8369:39195 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T22:49:24,415 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36871 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd96e8dc8369,45185,1732402163910 2024-11-23T22:49:24,415 DEBUG [RS:0;dd96e8dc8369:36415 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da 2024-11-23T22:49:24,415 DEBUG [RS:0;dd96e8dc8369:36415 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42261 2024-11-23T22:49:24,415 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36871 {}] master.ServerManager(517): Registering regionserver=dd96e8dc8369,45185,1732402163910 2024-11-23T22:49:24,415 DEBUG [RS:0;dd96e8dc8369:36415 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T22:49:24,418 DEBUG [RS:2;dd96e8dc8369:45185 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da 2024-11-23T22:49:24,418 DEBUG [RS:2;dd96e8dc8369:45185 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42261 2024-11-23T22:49:24,418 DEBUG [RS:2;dd96e8dc8369:45185 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T22:49:24,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T22:49:24,465 DEBUG [RS:1;dd96e8dc8369:39195 {}] zookeeper.ZKUtil(111): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd96e8dc8369,39195,1732402163849 2024-11-23T22:49:24,465 WARN [RS:1;dd96e8dc8369:39195 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T22:49:24,465 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd96e8dc8369,39195,1732402163849] 2024-11-23T22:49:24,465 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd96e8dc8369,45185,1732402163910] 2024-11-23T22:49:24,466 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd96e8dc8369,36415,1732402163807] 2024-11-23T22:49:24,466 INFO [RS:1;dd96e8dc8369:39195 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-23T22:49:24,466 DEBUG [RS:0;dd96e8dc8369:36415 {}] zookeeper.ZKUtil(111): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd96e8dc8369,36415,1732402163807 2024-11-23T22:49:24,466 DEBUG [RS:2;dd96e8dc8369:45185 {}] zookeeper.ZKUtil(111): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd96e8dc8369,45185,1732402163910 2024-11-23T22:49:24,466 WARN [RS:0;dd96e8dc8369:36415 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T22:49:24,466 DEBUG [RS:1;dd96e8dc8369:39195 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/WALs/dd96e8dc8369,39195,1732402163849 2024-11-23T22:49:24,466 WARN [RS:2;dd96e8dc8369:45185 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T22:49:24,466 INFO [RS:0;dd96e8dc8369:36415 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-23T22:49:24,466 INFO [RS:2;dd96e8dc8369:45185 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-23T22:49:24,466 DEBUG [RS:0;dd96e8dc8369:36415 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/WALs/dd96e8dc8369,36415,1732402163807 2024-11-23T22:49:24,466 DEBUG [RS:2;dd96e8dc8369:45185 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/WALs/dd96e8dc8369,45185,1732402163910 2024-11-23T22:49:24,470 INFO [RS:0;dd96e8dc8369:36415 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T22:49:24,470 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T22:49:24,470 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T22:49:24,475 INFO [RS:0;dd96e8dc8369:36415 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T22:49:24,475 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T22:49:24,476 INFO [RS:0;dd96e8dc8369:36415 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T22:49:24,476 INFO [RS:1;dd96e8dc8369:39195 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T22:49:24,476 INFO [RS:0;dd96e8dc8369:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,476 INFO [RS:1;dd96e8dc8369:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,476 INFO [RS:0;dd96e8dc8369:36415 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T22:49:24,476 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T22:49:24,477 INFO [RS:0;dd96e8dc8369:36415 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T22:49:24,477 INFO [RS:1;dd96e8dc8369:39195 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T22:49:24,477 INFO [RS:0;dd96e8dc8369:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,477 INFO [RS:1;dd96e8dc8369:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,477 DEBUG [RS:0;dd96e8dc8369:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,477 DEBUG [RS:1;dd96e8dc8369:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,477 DEBUG [RS:0;dd96e8dc8369:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,477 DEBUG [RS:1;dd96e8dc8369:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,478 DEBUG [RS:0;dd96e8dc8369:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,478 DEBUG [RS:1;dd96e8dc8369:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,478 DEBUG [RS:0;dd96e8dc8369:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,478 DEBUG [RS:1;dd96e8dc8369:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,478 DEBUG [RS:0;dd96e8dc8369:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,478 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T22:49:24,478 DEBUG [RS:1;dd96e8dc8369:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,478 DEBUG [RS:0;dd96e8dc8369:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2 2024-11-23T22:49:24,478 DEBUG [RS:1;dd96e8dc8369:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2 2024-11-23T22:49:24,478 DEBUG [RS:0;dd96e8dc8369:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,478 DEBUG [RS:1;dd96e8dc8369:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,478 DEBUG [RS:0;dd96e8dc8369:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,478 DEBUG [RS:1;dd96e8dc8369:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,478 DEBUG [RS:0;dd96e8dc8369:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,478 INFO [RS:2;dd96e8dc8369:45185 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T22:49:24,478 DEBUG [RS:1;dd96e8dc8369:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,478 INFO [RS:2;dd96e8dc8369:45185 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,478 DEBUG [RS:0;dd96e8dc8369:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,478 DEBUG [RS:1;dd96e8dc8369:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,478 DEBUG [RS:0;dd96e8dc8369:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,478 DEBUG [RS:1;dd96e8dc8369:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,479 DEBUG [RS:0;dd96e8dc8369:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,479 DEBUG [RS:1;dd96e8dc8369:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,479 DEBUG [RS:0;dd96e8dc8369:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:49:24,479 DEBUG [RS:1;dd96e8dc8369:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:49:24,479 DEBUG [RS:0;dd96e8dc8369:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:49:24,479 DEBUG [RS:1;dd96e8dc8369:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:49:24,481 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T22:49:24,482 INFO [RS:2;dd96e8dc8369:45185 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T22:49:24,482 INFO [RS:2;dd96e8dc8369:45185 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,482 DEBUG [RS:2;dd96e8dc8369:45185 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,483 DEBUG [RS:2;dd96e8dc8369:45185 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,483 DEBUG [RS:2;dd96e8dc8369:45185 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,483 DEBUG [RS:2;dd96e8dc8369:45185 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,483 DEBUG [RS:2;dd96e8dc8369:45185 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,483 DEBUG [RS:2;dd96e8dc8369:45185 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2 2024-11-23T22:49:24,483 DEBUG [RS:2;dd96e8dc8369:45185 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,483 DEBUG [RS:2;dd96e8dc8369:45185 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,483 DEBUG [RS:2;dd96e8dc8369:45185 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,483 DEBUG [RS:2;dd96e8dc8369:45185 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,483 DEBUG [RS:2;dd96e8dc8369:45185 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,483 DEBUG [RS:2;dd96e8dc8369:45185 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:49:24,483 DEBUG [RS:2;dd96e8dc8369:45185 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:49:24,484 DEBUG [RS:2;dd96e8dc8369:45185 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:49:24,491 INFO [RS:0;dd96e8dc8369:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,491 INFO [RS:1;dd96e8dc8369:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,491 INFO [RS:0;dd96e8dc8369:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,492 INFO [RS:1;dd96e8dc8369:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,492 INFO [RS:0;dd96e8dc8369:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,492 INFO [RS:0;dd96e8dc8369:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,492 INFO [RS:1;dd96e8dc8369:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,492 INFO [RS:0;dd96e8dc8369:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,492 INFO [RS:1;dd96e8dc8369:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,492 INFO [RS:0;dd96e8dc8369:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,36415,1732402163807-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T22:49:24,492 INFO [RS:1;dd96e8dc8369:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,492 INFO [RS:2;dd96e8dc8369:45185 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,492 INFO [RS:1;dd96e8dc8369:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,39195,1732402163849-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T22:49:24,492 INFO [RS:2;dd96e8dc8369:45185 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,492 INFO [RS:2;dd96e8dc8369:45185 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,492 INFO [RS:2;dd96e8dc8369:45185 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,492 INFO [RS:2;dd96e8dc8369:45185 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,492 INFO [RS:2;dd96e8dc8369:45185 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,45185,1732402163910-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T22:49:24,506 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T22:49:24,506 INFO [RS:0;dd96e8dc8369:36415 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T22:49:24,507 INFO [RS:2;dd96e8dc8369:45185 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,45185,1732402163910-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,507 INFO [RS:0;dd96e8dc8369:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,36415,1732402163807-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,507 INFO [RS:2;dd96e8dc8369:45185 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,507 INFO [RS:0;dd96e8dc8369:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,507 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.Replication(171): dd96e8dc8369,45185,1732402163910 started 2024-11-23T22:49:24,507 INFO [RS:0;dd96e8dc8369:36415 {}] regionserver.Replication(171): dd96e8dc8369,36415,1732402163807 started 2024-11-23T22:49:24,509 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T22:49:24,510 INFO [RS:1;dd96e8dc8369:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,39195,1732402163849-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,510 INFO [RS:1;dd96e8dc8369:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,510 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.Replication(171): dd96e8dc8369,39195,1732402163849 started 2024-11-23T22:49:24,512 WARN [dd96e8dc8369:36871 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T22:49:24,520 INFO [RS:2;dd96e8dc8369:45185 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,520 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.HRegionServer(1482): Serving as dd96e8dc8369,45185,1732402163910, RpcServer on dd96e8dc8369/172.17.0.2:45185, sessionid=0x10169e32b2b0003 2024-11-23T22:49:24,521 DEBUG [RS:2;dd96e8dc8369:45185 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T22:49:24,521 DEBUG [RS:2;dd96e8dc8369:45185 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd96e8dc8369,45185,1732402163910 2024-11-23T22:49:24,521 DEBUG [RS:2;dd96e8dc8369:45185 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,45185,1732402163910' 2024-11-23T22:49:24,521 DEBUG [RS:2;dd96e8dc8369:45185 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T22:49:24,521 INFO [RS:0;dd96e8dc8369:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,521 INFO [RS:0;dd96e8dc8369:36415 {}] regionserver.HRegionServer(1482): Serving as dd96e8dc8369,36415,1732402163807, RpcServer on dd96e8dc8369/172.17.0.2:36415, sessionid=0x10169e32b2b0001 2024-11-23T22:49:24,521 DEBUG [RS:0;dd96e8dc8369:36415 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T22:49:24,521 DEBUG [RS:0;dd96e8dc8369:36415 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd96e8dc8369,36415,1732402163807 2024-11-23T22:49:24,521 DEBUG [RS:0;dd96e8dc8369:36415 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,36415,1732402163807' 2024-11-23T22:49:24,521 DEBUG [RS:0;dd96e8dc8369:36415 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T22:49:24,521 DEBUG [RS:2;dd96e8dc8369:45185 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T22:49:24,522 DEBUG [RS:0;dd96e8dc8369:36415 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T22:49:24,522 DEBUG [RS:2;dd96e8dc8369:45185 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T22:49:24,522 DEBUG [RS:2;dd96e8dc8369:45185 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T22:49:24,522 DEBUG [RS:2;dd96e8dc8369:45185 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd96e8dc8369,45185,1732402163910 2024-11-23T22:49:24,522 DEBUG [RS:0;dd96e8dc8369:36415 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T22:49:24,522 DEBUG [RS:2;dd96e8dc8369:45185 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,45185,1732402163910' 2024-11-23T22:49:24,522 DEBUG [RS:0;dd96e8dc8369:36415 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T22:49:24,522 DEBUG [RS:2;dd96e8dc8369:45185 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T22:49:24,522 DEBUG [RS:0;dd96e8dc8369:36415 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd96e8dc8369,36415,1732402163807 2024-11-23T22:49:24,522 DEBUG [RS:0;dd96e8dc8369:36415 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,36415,1732402163807' 2024-11-23T22:49:24,522 DEBUG [RS:0;dd96e8dc8369:36415 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T22:49:24,523 DEBUG [RS:2;dd96e8dc8369:45185 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T22:49:24,523 DEBUG [RS:0;dd96e8dc8369:36415 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T22:49:24,523 DEBUG [RS:0;dd96e8dc8369:36415 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T22:49:24,523 INFO [RS:0;dd96e8dc8369:36415 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T22:49:24,523 DEBUG [RS:2;dd96e8dc8369:45185 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T22:49:24,523 INFO [RS:0;dd96e8dc8369:36415 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T22:49:24,523 INFO [RS:1;dd96e8dc8369:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:24,523 INFO [RS:2;dd96e8dc8369:45185 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T22:49:24,523 INFO [RS:2;dd96e8dc8369:45185 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T22:49:24,523 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.HRegionServer(1482): Serving as dd96e8dc8369,39195,1732402163849, RpcServer on dd96e8dc8369/172.17.0.2:39195, sessionid=0x10169e32b2b0002 2024-11-23T22:49:24,523 DEBUG [RS:1;dd96e8dc8369:39195 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T22:49:24,523 DEBUG [RS:1;dd96e8dc8369:39195 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd96e8dc8369,39195,1732402163849 2024-11-23T22:49:24,524 DEBUG [RS:1;dd96e8dc8369:39195 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,39195,1732402163849' 2024-11-23T22:49:24,524 DEBUG [RS:1;dd96e8dc8369:39195 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T22:49:24,524 DEBUG [RS:1;dd96e8dc8369:39195 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T22:49:24,524 DEBUG [RS:1;dd96e8dc8369:39195 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T22:49:24,524 DEBUG [RS:1;dd96e8dc8369:39195 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T22:49:24,525 DEBUG [RS:1;dd96e8dc8369:39195 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd96e8dc8369,39195,1732402163849 2024-11-23T22:49:24,525 DEBUG [RS:1;dd96e8dc8369:39195 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,39195,1732402163849' 2024-11-23T22:49:24,525 DEBUG [RS:1;dd96e8dc8369:39195 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T22:49:24,525 DEBUG [RS:1;dd96e8dc8369:39195 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T22:49:24,525 DEBUG [RS:1;dd96e8dc8369:39195 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T22:49:24,525 INFO [RS:1;dd96e8dc8369:39195 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T22:49:24,525 INFO [RS:1;dd96e8dc8369:39195 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T22:49:24,628 INFO [RS:0;dd96e8dc8369:36415 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C36415%2C1732402163807, suffix=, logDir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/WALs/dd96e8dc8369,36415,1732402163807, archiveDir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/oldWALs, maxLogs=32 2024-11-23T22:49:24,628 INFO [RS:2;dd96e8dc8369:45185 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C45185%2C1732402163910, suffix=, logDir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/WALs/dd96e8dc8369,45185,1732402163910, archiveDir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/oldWALs, maxLogs=32 2024-11-23T22:49:24,629 INFO [RS:1;dd96e8dc8369:39195 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C39195%2C1732402163849, suffix=, logDir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/WALs/dd96e8dc8369,39195,1732402163849, archiveDir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/oldWALs, maxLogs=32 2024-11-23T22:49:24,635 INFO [RS:1;dd96e8dc8369:39195 {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C39195%2C1732402163849.1732402164634 2024-11-23T22:49:24,635 INFO [RS:2;dd96e8dc8369:45185 {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C45185%2C1732402163910.1732402164634 2024-11-23T22:49:24,635 INFO [RS:0;dd96e8dc8369:36415 {}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C36415%2C1732402163807.1732402164634 2024-11-23T22:49:24,650 INFO [RS:0;dd96e8dc8369:36415 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/WALs/dd96e8dc8369,36415,1732402163807/dd96e8dc8369%2C36415%2C1732402163807.1732402164634 2024-11-23T22:49:24,650 INFO [RS:1;dd96e8dc8369:39195 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/WALs/dd96e8dc8369,39195,1732402163849/dd96e8dc8369%2C39195%2C1732402163849.1732402164634 2024-11-23T22:49:24,650 INFO [RS:2;dd96e8dc8369:45185 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/WALs/dd96e8dc8369,45185,1732402163910/dd96e8dc8369%2C45185%2C1732402163910.1732402164634 2024-11-23T22:49:24,661 DEBUG [RS:1;dd96e8dc8369:39195 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43843:43843),(127.0.0.1/127.0.0.1:34501:34501),(127.0.0.1/127.0.0.1:36221:36221)] 2024-11-23T22:49:24,662 DEBUG [RS:2;dd96e8dc8369:45185 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43843:43843),(127.0.0.1/127.0.0.1:36221:36221),(127.0.0.1/127.0.0.1:34501:34501)] 2024-11-23T22:49:24,662 DEBUG [RS:0;dd96e8dc8369:36415 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43843:43843),(127.0.0.1/127.0.0.1:34501:34501),(127.0.0.1/127.0.0.1:36221:36221)] 2024-11-23T22:49:24,763 DEBUG [dd96e8dc8369:36871 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-23T22:49:24,763 DEBUG [dd96e8dc8369:36871 {}] balancer.BalancerClusterState(204): Hosts are {dd96e8dc8369=0} racks are {/default-rack=0} 2024-11-23T22:49:24,765 DEBUG [dd96e8dc8369:36871 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T22:49:24,765 DEBUG [dd96e8dc8369:36871 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T22:49:24,765 DEBUG [dd96e8dc8369:36871 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-23T22:49:24,765 DEBUG [dd96e8dc8369:36871 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T22:49:24,766 DEBUG [dd96e8dc8369:36871 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T22:49:24,766 DEBUG [dd96e8dc8369:36871 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-23T22:49:24,766 INFO [dd96e8dc8369:36871 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T22:49:24,766 INFO [dd96e8dc8369:36871 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T22:49:24,766 INFO [dd96e8dc8369:36871 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-23T22:49:24,766 DEBUG [dd96e8dc8369:36871 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T22:49:24,766 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=dd96e8dc8369,45185,1732402163910 2024-11-23T22:49:24,768 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,45185,1732402163910, state=OPENING 2024-11-23T22:49:24,816 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T22:49:24,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:24,827 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T22:49:24,827 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:49:24,828 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:49:24,828 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,45185,1732402163910}] 2024-11-23T22:49:24,828 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:49:24,828 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:49:24,982 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T22:49:24,984 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43349, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T22:49:24,988 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T22:49:24,989 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-23T22:49:24,991 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C45185%2C1732402163910.meta, suffix=.meta, logDir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/WALs/dd96e8dc8369,45185,1732402163910, archiveDir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/oldWALs, maxLogs=32 2024-11-23T22:49:24,993 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor dd96e8dc8369%2C45185%2C1732402163910.meta.1732402164992.meta 2024-11-23T22:49:25,002 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/WALs/dd96e8dc8369,45185,1732402163910/dd96e8dc8369%2C45185%2C1732402163910.meta.1732402164992.meta 2024-11-23T22:49:25,004 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43843:43843),(127.0.0.1/127.0.0.1:34501:34501),(127.0.0.1/127.0.0.1:36221:36221)] 2024-11-23T22:49:25,005 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T22:49:25,005 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T22:49:25,005 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T22:49:25,006 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T22:49:25,006 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T22:49:25,006 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:49:25,006 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T22:49:25,006 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T22:49:25,008 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T22:49:25,009 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T22:49:25,009 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:25,010 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:49:25,010 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T22:49:25,011 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T22:49:25,011 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:25,011 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:49:25,012 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T22:49:25,013 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T22:49:25,013 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:25,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:49:25,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T22:49:25,014 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T22:49:25,014 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:25,015 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:49:25,015 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T22:49:25,016 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740 2024-11-23T22:49:25,018 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740 2024-11-23T22:49:25,019 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T22:49:25,019 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T22:49:25,020 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T22:49:25,022 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T22:49:25,023 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74064714, jitterRate=0.10365024209022522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T22:49:25,024 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T22:49:25,025 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732402165006Writing region info on filesystem at 1732402165006Initializing all the Stores at 1732402165007 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402165008 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402165008Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402165008Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732402165008Cleaning up temporary data from old regions at 1732402165019 (+11 ms)Running coprocessor post-open hooks at 1732402165024 (+5 ms)Region opened successfully at 1732402165025 (+1 ms) 2024-11-23T22:49:25,027 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732402164982 2024-11-23T22:49:25,030 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T22:49:25,030 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T22:49:25,031 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,45185,1732402163910 2024-11-23T22:49:25,033 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,45185,1732402163910, state=OPEN 2024-11-23T22:49:25,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T22:49:25,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T22:49:25,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T22:49:25,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T22:49:25,046 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=dd96e8dc8369,45185,1732402163910 2024-11-23T22:49:25,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:49:25,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:49:25,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:49:25,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:49:25,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T22:49:25,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,45185,1732402163910 in 219 msec 2024-11-23T22:49:25,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T22:49:25,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 693 msec 2024-11-23T22:49:25,056 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T22:49:25,056 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T22:49:25,058 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T22:49:25,058 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,45185,1732402163910, seqNum=-1] 2024-11-23T22:49:25,058 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T22:49:25,060 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32829, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T22:49:25,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 762 msec 2024-11-23T22:49:25,069 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732402165069, completionTime=-1 2024-11-23T22:49:25,070 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-23T22:49:25,070 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T22:49:25,075 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-23T22:49:25,075 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732402225075 2024-11-23T22:49:25,075 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732402285075 2024-11-23T22:49:25,075 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-23T22:49:25,087 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,36871,1732402163604-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:25,087 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,36871,1732402163604-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:25,087 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,36871,1732402163604-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:25,087 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-dd96e8dc8369:36871, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:25,087 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:25,088 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:25,090 DEBUG [master/dd96e8dc8369:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T22:49:25,093 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.108sec 2024-11-23T22:49:25,093 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T22:49:25,093 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T22:49:25,094 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T22:49:25,094 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T22:49:25,094 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T22:49:25,094 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,36871,1732402163604-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T22:49:25,094 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,36871,1732402163604-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T22:49:25,097 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T22:49:25,097 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T22:49:25,097 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,36871,1732402163604-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:49:25,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55afdbba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T22:49:25,144 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request dd96e8dc8369,36871,-1 for getting cluster id 2024-11-23T22:49:25,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T22:49:25,145 DEBUG [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '98b63b6d-26a6-4031-a18a-9eb247a13068' 2024-11-23T22:49:25,146 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T22:49:25,146 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "98b63b6d-26a6-4031-a18a-9eb247a13068" 2024-11-23T22:49:25,146 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47953c78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T22:49:25,146 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [dd96e8dc8369,36871,-1] 2024-11-23T22:49:25,146 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T22:49:25,147 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:25,148 INFO [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T22:49:25,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74e402b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T22:49:25,150 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T22:49:25,151 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,45185,1732402163910, seqNum=-1] 2024-11-23T22:49:25,151 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T22:49:25,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T22:49:25,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=dd96e8dc8369,36871,1732402163604 2024-11-23T22:49:25,156 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T22:49:25,158 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncConnectionImpl(321): The fetched master address is dd96e8dc8369,36871,1732402163604 2024-11-23T22:49:25,158 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7b9b7611 2024-11-23T22:49:25,158 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T22:49:25,160 INFO [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T22:49:25,161 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36871 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestHBaseWalOnEC', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T22:49:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36871 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestHBaseWalOnEC 2024-11-23T22:49:25,164 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T22:49:25,165 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:25,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36871 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestHBaseWalOnEC" procId is: 4 2024-11-23T22:49:25,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36871 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T22:49:25,166 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T22:49:25,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741837_1013 (size=392) 2024-11-23T22:49:25,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741837_1013 (size=392) 2024-11-23T22:49:25,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41827 is added to blk_1073741837_1013 (size=392) 2024-11-23T22:49:25,177 INFO [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 570f85a0cc0154d79b758f945f8bf044, NAME => 'TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestHBaseWalOnEC', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da 2024-11-23T22:49:25,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741838_1014 (size=51) 2024-11-23T22:49:25,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741838_1014 (size=51) 2024-11-23T22:49:25,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41827 is added to blk_1073741838_1014 (size=51) 2024-11-23T22:49:25,190 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(898): Instantiated TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:49:25,190 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1722): Closing 570f85a0cc0154d79b758f945f8bf044, disabling compactions & flushes 2024-11-23T22:49:25,191 INFO [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1755): Closing region TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044. 2024-11-23T22:49:25,191 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044. 2024-11-23T22:49:25,191 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044. after waiting 0 ms 2024-11-23T22:49:25,191 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044. 2024-11-23T22:49:25,191 INFO [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1973): Closed TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044. 2024-11-23T22:49:25,191 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1676): Region close journal for 570f85a0cc0154d79b758f945f8bf044: Waiting for close lock at 1732402165190Disabling compacts and flushes for region at 1732402165190Disabling writes for close at 1732402165191 (+1 ms)Writing region close event to WAL at 1732402165191Closed at 1732402165191 2024-11-23T22:49:25,193 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T22:49:25,193 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044.","families":{"info":[{"qualifier":"regioninfo","vlen":50,"tag":[],"timestamp":"1732402165193"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732402165193"}]},"ts":"1732402165193"} 2024-11-23T22:49:25,196 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T22:49:25,198 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T22:49:25,198 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestHBaseWalOnEC","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732402165198"}]},"ts":"1732402165198"} 2024-11-23T22:49:25,201 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestHBaseWalOnEC, state=ENABLING in hbase:meta 2024-11-23T22:49:25,202 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {dd96e8dc8369=0} racks are {/default-rack=0} 2024-11-23T22:49:25,203 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T22:49:25,203 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T22:49:25,203 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-23T22:49:25,203 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T22:49:25,203 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T22:49:25,203 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-23T22:49:25,203 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T22:49:25,203 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T22:49:25,203 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-23T22:49:25,203 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T22:49:25,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=570f85a0cc0154d79b758f945f8bf044, ASSIGN}] 2024-11-23T22:49:25,205 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=570f85a0cc0154d79b758f945f8bf044, ASSIGN 2024-11-23T22:49:25,207 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=570f85a0cc0154d79b758f945f8bf044, ASSIGN; state=OFFLINE, location=dd96e8dc8369,39195,1732402163849; forceNewPlan=false, retain=false 2024-11-23T22:49:25,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36871 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T22:49:25,357 INFO [dd96e8dc8369:36871 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T22:49:25,358 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=570f85a0cc0154d79b758f945f8bf044, regionState=OPENING, regionLocation=dd96e8dc8369,39195,1732402163849 2024-11-23T22:49:25,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=570f85a0cc0154d79b758f945f8bf044, ASSIGN because future has completed 2024-11-23T22:49:25,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 570f85a0cc0154d79b758f945f8bf044, server=dd96e8dc8369,39195,1732402163849}] 2024-11-23T22:49:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36871 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T22:49:25,522 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T22:49:25,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57161, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T22:49:25,528 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044. 2024-11-23T22:49:25,528 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 570f85a0cc0154d79b758f945f8bf044, NAME => 'TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044.', STARTKEY => '', ENDKEY => ''} 2024-11-23T22:49:25,529 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestHBaseWalOnEC 570f85a0cc0154d79b758f945f8bf044 2024-11-23T22:49:25,529 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:49:25,529 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 570f85a0cc0154d79b758f945f8bf044 2024-11-23T22:49:25,529 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 570f85a0cc0154d79b758f945f8bf044 2024-11-23T22:49:25,531 INFO [StoreOpener-570f85a0cc0154d79b758f945f8bf044-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region 570f85a0cc0154d79b758f945f8bf044 2024-11-23T22:49:25,532 INFO [StoreOpener-570f85a0cc0154d79b758f945f8bf044-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 570f85a0cc0154d79b758f945f8bf044 columnFamilyName cf 2024-11-23T22:49:25,532 DEBUG [StoreOpener-570f85a0cc0154d79b758f945f8bf044-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:49:25,533 INFO [StoreOpener-570f85a0cc0154d79b758f945f8bf044-1 {}] regionserver.HStore(327): Store=570f85a0cc0154d79b758f945f8bf044/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T22:49:25,533 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 570f85a0cc0154d79b758f945f8bf044 2024-11-23T22:49:25,534 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/default/TestHBaseWalOnEC/570f85a0cc0154d79b758f945f8bf044 2024-11-23T22:49:25,535 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/default/TestHBaseWalOnEC/570f85a0cc0154d79b758f945f8bf044 2024-11-23T22:49:25,535 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 570f85a0cc0154d79b758f945f8bf044 2024-11-23T22:49:25,535 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 570f85a0cc0154d79b758f945f8bf044 2024-11-23T22:49:25,537 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 570f85a0cc0154d79b758f945f8bf044 2024-11-23T22:49:25,540 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/default/TestHBaseWalOnEC/570f85a0cc0154d79b758f945f8bf044/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T22:49:25,540 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 570f85a0cc0154d79b758f945f8bf044; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66695640, jitterRate=-0.006157517433166504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T22:49:25,541 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 570f85a0cc0154d79b758f945f8bf044 2024-11-23T22:49:25,541 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 570f85a0cc0154d79b758f945f8bf044: Running coprocessor pre-open hook at 1732402165529Writing region info on filesystem at 1732402165529Initializing all the Stores at 1732402165530 (+1 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732402165530Cleaning up temporary data from old regions at 1732402165535 (+5 ms)Running coprocessor post-open hooks at 1732402165541 (+6 ms)Region opened successfully at 1732402165541 2024-11-23T22:49:25,543 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044., pid=6, masterSystemTime=1732402165522 2024-11-23T22:49:25,546 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044. 2024-11-23T22:49:25,546 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044. 2024-11-23T22:49:25,547 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=570f85a0cc0154d79b758f945f8bf044, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,39195,1732402163849 2024-11-23T22:49:25,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 570f85a0cc0154d79b758f945f8bf044, server=dd96e8dc8369,39195,1732402163849 because future has completed 2024-11-23T22:49:25,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T22:49:25,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 570f85a0cc0154d79b758f945f8bf044, server=dd96e8dc8369,39195,1732402163849 in 187 msec 2024-11-23T22:49:25,560 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T22:49:25,560 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=570f85a0cc0154d79b758f945f8bf044, ASSIGN in 353 msec 2024-11-23T22:49:25,562 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T22:49:25,562 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestHBaseWalOnEC","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732402165562"}]},"ts":"1732402165562"} 2024-11-23T22:49:25,565 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestHBaseWalOnEC, state=ENABLED in hbase:meta 2024-11-23T22:49:25,567 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T22:49:25,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestHBaseWalOnEC in 406 msec 2024-11-23T22:49:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36871 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T22:49:25,798 INFO [RPCClient-NioEventLoopGroup-6-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestHBaseWalOnEC completed 2024-11-23T22:49:25,798 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table TestHBaseWalOnEC get assigned. Timeout = 60000ms 2024-11-23T22:49:25,798 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T22:49:25,804 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table TestHBaseWalOnEC assigned to meta. Checking AM states. 2024-11-23T22:49:25,804 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T22:49:25,804 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table TestHBaseWalOnEC assigned. 2024-11-23T22:49:25,808 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestHBaseWalOnEC', row='row', locateType=CURRENT is [region=TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044., hostname=dd96e8dc8369,39195,1732402163849, seqNum=2] 2024-11-23T22:49:25,809 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T22:49:25,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T22:49:25,814 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36871 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestHBaseWalOnEC 2024-11-23T22:49:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36871 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC 2024-11-23T22:49:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36871 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T22:49:25,818 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC execute state=FLUSH_TABLE_PREPARE 2024-11-23T22:49:25,820 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T22:49:25,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T22:49:25,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36871 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T22:49:25,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39195 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T22:49:25,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044. 2024-11-23T22:49:25,977 INFO [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 570f85a0cc0154d79b758f945f8bf044 1/1 column families, dataSize=32 B heapSize=360 B 2024-11-23T22:49:26,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/default/TestHBaseWalOnEC/570f85a0cc0154d79b758f945f8bf044/.tmp/cf/c069bb62c3ee42aeb6f09b28df5a9dd3 is 36, key is row/cf:cq/1732402165812/Put/seqid=0 2024-11-23T22:49:26,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741839_1015 (size=4787) 2024-11-23T22:49:26,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741839_1015 (size=4787) 2024-11-23T22:49:26,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41827 is added to blk_1073741839_1015 (size=4787) 2024-11-23T22:49:26,008 INFO [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/default/TestHBaseWalOnEC/570f85a0cc0154d79b758f945f8bf044/.tmp/cf/c069bb62c3ee42aeb6f09b28df5a9dd3 2024-11-23T22:49:26,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/default/TestHBaseWalOnEC/570f85a0cc0154d79b758f945f8bf044/.tmp/cf/c069bb62c3ee42aeb6f09b28df5a9dd3 as hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/default/TestHBaseWalOnEC/570f85a0cc0154d79b758f945f8bf044/cf/c069bb62c3ee42aeb6f09b28df5a9dd3 2024-11-23T22:49:26,025 INFO [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/default/TestHBaseWalOnEC/570f85a0cc0154d79b758f945f8bf044/cf/c069bb62c3ee42aeb6f09b28df5a9dd3, entries=1, sequenceid=5, filesize=4.7 K 2024-11-23T22:49:26,027 INFO [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~32 B/32, heapSize ~344 B/344, currentSize=0 B/0 for 570f85a0cc0154d79b758f945f8bf044 in 51ms, sequenceid=5, compaction requested=false 2024-11-23T22:49:26,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 570f85a0cc0154d79b758f945f8bf044: 2024-11-23T22:49:26,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044. 2024-11-23T22:49:26,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-23T22:49:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36871 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-23T22:49:26,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T22:49:26,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 209 msec 2024-11-23T22:49:26,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC in 219 msec 2024-11-23T22:49:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36871 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T22:49:26,137 INFO [RPCClient-NioEventLoopGroup-6-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestHBaseWalOnEC completed 2024-11-23T22:49:26,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T22:49:26,144 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T22:49:26,144 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.TestHBaseWalOnEC.tearDown(TestHBaseWalOnEC.java:101) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T22:49:26,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:26,145 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:26,145 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T22:49:26,145 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T22:49:26,145 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1937593665, stopped=false 2024-11-23T22:49:26,145 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=dd96e8dc8369,36871,1732402163604 2024-11-23T22:49:26,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T22:49:26,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T22:49:26,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T22:49:26,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T22:49:26,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:26,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:26,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:26,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:26,207 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T22:49:26,207 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T22:49:26,208 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:49:26,208 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.TestHBaseWalOnEC.tearDown(TestHBaseWalOnEC.java:101) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T22:49:26,208 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:49:26,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:26,209 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:49:26,209 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:49:26,209 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd96e8dc8369,36415,1732402163807' ***** 2024-11-23T22:49:26,209 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T22:49:26,210 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd96e8dc8369,39195,1732402163849' ***** 2024-11-23T22:49:26,210 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T22:49:26,210 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd96e8dc8369,45185,1732402163910' ***** 2024-11-23T22:49:26,210 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T22:49:26,210 INFO [RS:0;dd96e8dc8369:36415 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T22:49:26,210 INFO [RS:0;dd96e8dc8369:36415 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T22:49:26,210 INFO [RS:0;dd96e8dc8369:36415 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T22:49:26,210 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T22:49:26,210 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T22:49:26,210 INFO [RS:0;dd96e8dc8369:36415 {}] regionserver.HRegionServer(959): stopping server dd96e8dc8369,36415,1732402163807 2024-11-23T22:49:26,211 INFO [RS:1;dd96e8dc8369:39195 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T22:49:26,211 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T22:49:26,211 INFO [RS:1;dd96e8dc8369:39195 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T22:49:26,211 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T22:49:26,211 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.HRegionServer(3091): Received CLOSE for 570f85a0cc0154d79b758f945f8bf044 2024-11-23T22:49:26,210 INFO [RS:0;dd96e8dc8369:36415 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T22:49:26,211 INFO [RS:2;dd96e8dc8369:45185 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T22:49:26,211 INFO [RS:2;dd96e8dc8369:45185 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T22:49:26,211 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T22:49:26,211 INFO [RS:0;dd96e8dc8369:36415 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;dd96e8dc8369:36415. 2024-11-23T22:49:26,211 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.HRegionServer(959): stopping server dd96e8dc8369,45185,1732402163910 2024-11-23T22:49:26,211 INFO [RS:2;dd96e8dc8369:45185 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T22:49:26,211 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.HRegionServer(959): stopping server dd96e8dc8369,39195,1732402163849 2024-11-23T22:49:26,211 INFO [RS:1;dd96e8dc8369:39195 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T22:49:26,211 INFO [RS:2;dd96e8dc8369:45185 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;dd96e8dc8369:45185. 2024-11-23T22:49:26,212 INFO [RS:1;dd96e8dc8369:39195 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;dd96e8dc8369:39195. 2024-11-23T22:49:26,212 DEBUG [RS:0;dd96e8dc8369:36415 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T22:49:26,212 DEBUG [RS:0;dd96e8dc8369:36415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:26,212 DEBUG [RS:2;dd96e8dc8369:45185 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T22:49:26,212 DEBUG [RS:1;dd96e8dc8369:39195 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T22:49:26,212 DEBUG [RS:2;dd96e8dc8369:45185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:26,212 DEBUG [RS:1;dd96e8dc8369:39195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:26,212 INFO [RS:0;dd96e8dc8369:36415 {}] regionserver.HRegionServer(976): stopping server dd96e8dc8369,36415,1732402163807; all regions closed. 2024-11-23T22:49:26,212 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T22:49:26,212 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-23T22:49:26,212 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T22:49:26,212 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T22:49:26,212 DEBUG [RS:1;dd96e8dc8369:39195 {}] regionserver.HRegionServer(1325): Online Regions={570f85a0cc0154d79b758f945f8bf044=TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044.} 2024-11-23T22:49:26,212 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T22:49:26,212 DEBUG [RS:1;dd96e8dc8369:39195 {}] regionserver.HRegionServer(1351): Waiting on 570f85a0cc0154d79b758f945f8bf044 2024-11-23T22:49:26,212 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 570f85a0cc0154d79b758f945f8bf044, disabling compactions & flushes 2024-11-23T22:49:26,213 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-23T22:49:26,213 INFO [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044. 2024-11-23T22:49:26,213 DEBUG [RS:2;dd96e8dc8369:45185 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-23T22:49:26,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,213 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044. 2024-11-23T22:49:26,213 DEBUG [RS:2;dd96e8dc8369:45185 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T22:49:26,213 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T22:49:26,213 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044. after waiting 0 ms 2024-11-23T22:49:26,213 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T22:49:26,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,213 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T22:49:26,213 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044. 2024-11-23T22:49:26,213 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T22:49:26,213 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T22:49:26,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,213 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.34 KB heapSize=3.38 KB 2024-11-23T22:49:26,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41827 is added to blk_1073741833_1009 (size=93) 2024-11-23T22:49:26,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741833_1009 (size=93) 2024-11-23T22:49:26,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741833_1009 (size=93) 2024-11-23T22:49:26,219 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/default/TestHBaseWalOnEC/570f85a0cc0154d79b758f945f8bf044/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-23T22:49:26,220 INFO [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044. 2024-11-23T22:49:26,220 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 570f85a0cc0154d79b758f945f8bf044: Waiting for close lock at 1732402166212Running coprocessor pre-close hooks at 1732402166212Disabling compacts and flushes for region at 1732402166212Disabling writes for close at 1732402166213 (+1 ms)Writing region close event to WAL at 1732402166214 (+1 ms)Running coprocessor post-close hooks at 1732402166220 (+6 ms)Closed at 1732402166220 2024-11-23T22:49:26,220 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044. 2024-11-23T22:49:26,221 DEBUG [RS:0;dd96e8dc8369:36415 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/oldWALs 2024-11-23T22:49:26,221 INFO [RS:0;dd96e8dc8369:36415 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog dd96e8dc8369%2C36415%2C1732402163807:(num 1732402164634) 2024-11-23T22:49:26,221 DEBUG [RS:0;dd96e8dc8369:36415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:26,221 INFO [RS:0;dd96e8dc8369:36415 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:49:26,221 INFO [RS:0;dd96e8dc8369:36415 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T22:49:26,221 INFO [RS:0;dd96e8dc8369:36415 {}] hbase.ChoreService(370): Chore service for: regionserver/dd96e8dc8369:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T22:49:26,221 INFO [RS:0;dd96e8dc8369:36415 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T22:49:26,221 INFO [RS:0;dd96e8dc8369:36415 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T22:49:26,221 INFO [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T22:49:26,221 INFO [RS:0;dd96e8dc8369:36415 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T22:49:26,221 INFO [RS:0;dd96e8dc8369:36415 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T22:49:26,221 INFO [RS:0;dd96e8dc8369:36415 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36415 2024-11-23T22:49:26,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T22:49:26,227 INFO [RS:0;dd96e8dc8369:36415 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T22:49:26,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd96e8dc8369,36415,1732402163807 2024-11-23T22:49:26,232 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740/.tmp/info/08aba5beb3624770a8510e4b8cfee706 is 153, key is TestHBaseWalOnEC,,1732402165160.570f85a0cc0154d79b758f945f8bf044./info:regioninfo/1732402165547/Put/seqid=0 2024-11-23T22:49:26,233 WARN [IPC Server handler 4 on default port 42261 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-23T22:49:26,233 WARN [IPC Server handler 4 on default port 42261 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-23T22:49:26,233 WARN [IPC Server handler 4 on default port 42261 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-23T22:49:26,237 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd96e8dc8369,36415,1732402163807] 2024-11-23T22:49:26,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741840_1016 (size=6637) 2024-11-23T22:49:26,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741840_1016 (size=6637) 2024-11-23T22:49:26,239 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.18 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740/.tmp/info/08aba5beb3624770a8510e4b8cfee706 2024-11-23T22:49:26,248 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd96e8dc8369,36415,1732402163807 already deleted, retry=false 2024-11-23T22:49:26,248 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd96e8dc8369,36415,1732402163807 expired; onlineServers=2 2024-11-23T22:49:26,261 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740/.tmp/ns/b35d7796bb64495893db42e3a5d8f5e0 is 43, key is default/ns:d/1732402165061/Put/seqid=0 2024-11-23T22:49:26,262 WARN [IPC Server handler 0 on default port 42261 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-23T22:49:26,263 WARN [IPC Server handler 0 on default port 42261 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-23T22:49:26,263 WARN [IPC Server handler 0 on default port 42261 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-23T22:49:26,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741841_1017 (size=5153) 2024-11-23T22:49:26,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741841_1017 (size=5153) 2024-11-23T22:49:26,268 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740/.tmp/ns/b35d7796bb64495893db42e3a5d8f5e0 2024-11-23T22:49:26,291 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740/.tmp/table/9073153daad440e88b6d4e9920c6ac1f is 52, key is TestHBaseWalOnEC/table:state/1732402165562/Put/seqid=0 2024-11-23T22:49:26,292 WARN [IPC Server handler 2 on default port 42261 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-23T22:49:26,292 WARN [IPC Server handler 2 on default port 42261 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-23T22:49:26,292 WARN [IPC Server handler 2 on default port 42261 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-23T22:49:26,294 INFO [regionserver/dd96e8dc8369:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:49:26,294 INFO [regionserver/dd96e8dc8369:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:49:26,294 INFO [regionserver/dd96e8dc8369:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:49:26,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741842_1018 (size=5249) 2024-11-23T22:49:26,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741842_1018 (size=5249) 2024-11-23T22:49:26,297 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=96 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740/.tmp/table/9073153daad440e88b6d4e9920c6ac1f 2024-11-23T22:49:26,305 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740/.tmp/info/08aba5beb3624770a8510e4b8cfee706 as hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740/info/08aba5beb3624770a8510e4b8cfee706 2024-11-23T22:49:26,312 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740/info/08aba5beb3624770a8510e4b8cfee706, entries=10, sequenceid=11, filesize=6.5 K 2024-11-23T22:49:26,314 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740/.tmp/ns/b35d7796bb64495893db42e3a5d8f5e0 as hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740/ns/b35d7796bb64495893db42e3a5d8f5e0 2024-11-23T22:49:26,321 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740/ns/b35d7796bb64495893db42e3a5d8f5e0, entries=2, sequenceid=11, filesize=5.0 K 2024-11-23T22:49:26,322 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740/.tmp/table/9073153daad440e88b6d4e9920c6ac1f as hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740/table/9073153daad440e88b6d4e9920c6ac1f 2024-11-23T22:49:26,330 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740/table/9073153daad440e88b6d4e9920c6ac1f, entries=2, sequenceid=11, filesize=5.1 K 2024-11-23T22:49:26,331 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.34 KB/1377, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 118ms, sequenceid=11, compaction requested=false 2024-11-23T22:49:26,337 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-23T22:49:26,337 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T22:49:26,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:49:26,337 INFO [RS:0;dd96e8dc8369:36415 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T22:49:26,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x10169e32b2b0001, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:49:26,337 INFO [RS:0;dd96e8dc8369:36415 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd96e8dc8369,36415,1732402163807; zookeeper connection closed. 2024-11-23T22:49:26,337 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T22:49:26,338 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732402166213Running coprocessor pre-close hooks at 1732402166213Disabling compacts and flushes for region at 1732402166213Disabling writes for close at 1732402166213Obtaining lock to block concurrent updates at 1732402166213Preparing flush snapshotting stores in 1588230740 at 1732402166213Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1377, getHeapSize=3392, getOffHeapSize=0, getCellsCount=14 at 1732402166214 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732402166215 (+1 ms)Flushing 1588230740/info: creating writer at 1732402166215Flushing 1588230740/info: appending metadata at 1732402166231 (+16 ms)Flushing 1588230740/info: closing flushed file at 1732402166231Flushing 1588230740/ns: creating writer at 1732402166245 (+14 ms)Flushing 1588230740/ns: appending metadata at 1732402166261 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732402166261Flushing 1588230740/table: creating writer at 1732402166276 (+15 ms)Flushing 1588230740/table: appending metadata at 1732402166290 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732402166290Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13565f98: reopening flushed file at 1732402166304 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fbd29f0: reopening flushed file at 1732402166312 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a429fa7: reopening flushed file at 1732402166321 (+9 ms)Finished flush of dataSize ~1.34 KB/1377, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 118ms, sequenceid=11, compaction requested=false at 1732402166331 (+10 ms)Writing region close event to WAL at 1732402166332 (+1 ms)Running coprocessor post-close hooks at 1732402166337 (+5 ms)Closed at 1732402166337 2024-11-23T22:49:26,338 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6e3f7672 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6e3f7672 2024-11-23T22:49:26,338 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T22:49:26,412 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.HRegionServer(976): stopping server dd96e8dc8369,39195,1732402163849; all regions closed. 2024-11-23T22:49:26,413 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.HRegionServer(976): stopping server dd96e8dc8369,45185,1732402163910; all regions closed. 2024-11-23T22:49:26,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41827 is added to blk_1073741836_1012 (size=2751) 2024-11-23T22:49:26,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741836_1012 (size=2751) 2024-11-23T22:49:26,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741836_1012 (size=2751) 2024-11-23T22:49:26,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741834_1010 (size=1298) 2024-11-23T22:49:26,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41827 is added to blk_1073741834_1010 (size=1298) 2024-11-23T22:49:26,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741834_1010 (size=1298) 2024-11-23T22:49:26,420 DEBUG [RS:2;dd96e8dc8369:45185 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/oldWALs 2024-11-23T22:49:26,420 INFO [RS:2;dd96e8dc8369:45185 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog dd96e8dc8369%2C45185%2C1732402163910.meta:.meta(num 1732402164992) 2024-11-23T22:49:26,421 DEBUG [RS:1;dd96e8dc8369:39195 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/oldWALs 2024-11-23T22:49:26,421 INFO [RS:1;dd96e8dc8369:39195 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog dd96e8dc8369%2C39195%2C1732402163849:(num 1732402164634) 2024-11-23T22:49:26,421 DEBUG [RS:1;dd96e8dc8369:39195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:26,421 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:49:26,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,421 INFO [RS:1;dd96e8dc8369:39195 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T22:49:26,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,421 INFO [RS:1;dd96e8dc8369:39195 {}] hbase.ChoreService(370): Chore service for: regionserver/dd96e8dc8369:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T22:49:26,421 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T22:49:26,421 INFO [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T22:49:26,421 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T22:49:26,421 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T22:49:26,421 INFO [RS:1;dd96e8dc8369:39195 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T22:49:26,422 INFO [RS:1;dd96e8dc8369:39195 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39195 2024-11-23T22:49:26,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741835_1011 (size=93) 2024-11-23T22:49:26,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41827 is added to blk_1073741835_1011 (size=93) 2024-11-23T22:49:26,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741835_1011 (size=93) 2024-11-23T22:49:26,427 DEBUG [RS:2;dd96e8dc8369:45185 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/oldWALs 2024-11-23T22:49:26,428 INFO [RS:2;dd96e8dc8369:45185 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog dd96e8dc8369%2C45185%2C1732402163910:(num 1732402164634) 2024-11-23T22:49:26,428 DEBUG [RS:2;dd96e8dc8369:45185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:49:26,428 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:49:26,428 INFO [RS:2;dd96e8dc8369:45185 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T22:49:26,428 INFO [RS:2;dd96e8dc8369:45185 {}] hbase.ChoreService(370): Chore service for: regionserver/dd96e8dc8369:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T22:49:26,428 INFO [RS:2;dd96e8dc8369:45185 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T22:49:26,428 INFO [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T22:49:26,428 INFO [RS:2;dd96e8dc8369:45185 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45185 2024-11-23T22:49:26,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd96e8dc8369,39195,1732402163849 2024-11-23T22:49:26,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T22:49:26,469 INFO [RS:1;dd96e8dc8369:39195 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T22:49:26,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd96e8dc8369,45185,1732402163910 2024-11-23T22:49:26,479 INFO [RS:2;dd96e8dc8369:45185 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T22:49:26,490 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd96e8dc8369,39195,1732402163849] 2024-11-23T22:49:26,511 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd96e8dc8369,39195,1732402163849 already deleted, retry=false 2024-11-23T22:49:26,511 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd96e8dc8369,39195,1732402163849 expired; onlineServers=1 2024-11-23T22:49:26,511 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd96e8dc8369,45185,1732402163910] 2024-11-23T22:49:26,521 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd96e8dc8369,45185,1732402163910 already deleted, retry=false 2024-11-23T22:49:26,522 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd96e8dc8369,45185,1732402163910 expired; onlineServers=0 2024-11-23T22:49:26,522 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'dd96e8dc8369,36871,1732402163604' ***** 2024-11-23T22:49:26,522 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T22:49:26,522 INFO [M:0;dd96e8dc8369:36871 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T22:49:26,522 INFO [M:0;dd96e8dc8369:36871 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T22:49:26,522 DEBUG [M:0;dd96e8dc8369:36871 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T22:49:26,522 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T22:49:26,522 DEBUG [M:0;dd96e8dc8369:36871 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T22:49:26,522 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402164311 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732402164311,5,FailOnTimeoutGroup] 2024-11-23T22:49:26,522 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402164311 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732402164311,5,FailOnTimeoutGroup] 2024-11-23T22:49:26,522 INFO [M:0;dd96e8dc8369:36871 {}] hbase.ChoreService(370): Chore service for: master/dd96e8dc8369:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T22:49:26,522 INFO [M:0;dd96e8dc8369:36871 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T22:49:26,522 DEBUG [M:0;dd96e8dc8369:36871 {}] master.HMaster(1795): Stopping service threads 2024-11-23T22:49:26,523 INFO [M:0;dd96e8dc8369:36871 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T22:49:26,523 INFO [M:0;dd96e8dc8369:36871 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T22:49:26,523 INFO [M:0;dd96e8dc8369:36871 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T22:49:26,523 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T22:49:26,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T22:49:26,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:49:26,532 DEBUG [M:0;dd96e8dc8369:36871 {}] zookeeper.ZKUtil(347): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T22:49:26,532 WARN [M:0;dd96e8dc8369:36871 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T22:49:26,533 INFO [M:0;dd96e8dc8369:36871 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/.lastflushedseqids 2024-11-23T22:49:26,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741843_1019 (size=127) 2024-11-23T22:49:26,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41827 is added to blk_1073741843_1019 (size=127) 2024-11-23T22:49:26,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741843_1019 (size=127) 2024-11-23T22:49:26,555 INFO [M:0;dd96e8dc8369:36871 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T22:49:26,555 INFO [M:0;dd96e8dc8369:36871 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T22:49:26,556 DEBUG [M:0;dd96e8dc8369:36871 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T22:49:26,556 INFO [M:0;dd96e8dc8369:36871 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:49:26,556 DEBUG [M:0;dd96e8dc8369:36871 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:49:26,556 DEBUG [M:0;dd96e8dc8369:36871 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T22:49:26,556 DEBUG [M:0;dd96e8dc8369:36871 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:49:26,556 INFO [M:0;dd96e8dc8369:36871 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=26.81 KB heapSize=34.10 KB 2024-11-23T22:49:26,576 DEBUG [M:0;dd96e8dc8369:36871 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a4afa770a87c4f58aa9298d380297190 is 82, key is hbase:meta,,1/info:regioninfo/1732402165031/Put/seqid=0 2024-11-23T22:49:26,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741844_1020 (size=5672) 2024-11-23T22:49:26,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741844_1020 (size=5672) 2024-11-23T22:49:26,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41827 is added to blk_1073741844_1020 (size=5672) 2024-11-23T22:49:26,585 INFO [M:0;dd96e8dc8369:36871 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a4afa770a87c4f58aa9298d380297190 2024-11-23T22:49:26,590 INFO [RS:1;dd96e8dc8369:39195 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T22:49:26,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:49:26,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x10169e32b2b0002, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:49:26,590 INFO [RS:1;dd96e8dc8369:39195 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd96e8dc8369,39195,1732402163849; zookeeper connection closed. 2024-11-23T22:49:26,590 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@64a63369 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@64a63369 2024-11-23T22:49:26,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:49:26,601 INFO [RS:2;dd96e8dc8369:45185 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T22:49:26,601 INFO [RS:2;dd96e8dc8369:45185 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd96e8dc8369,45185,1732402163910; zookeeper connection closed. 2024-11-23T22:49:26,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45185-0x10169e32b2b0003, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:49:26,601 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@14c2371a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@14c2371a 2024-11-23T22:49:26,601 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-23T22:49:26,609 DEBUG [M:0;dd96e8dc8369:36871 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5911b946239e43928345f807ed267f6b is 747, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732402165568/Put/seqid=0 2024-11-23T22:49:26,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741845_1021 (size=6437) 2024-11-23T22:49:26,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41827 is added to blk_1073741845_1021 (size=6437) 2024-11-23T22:49:26,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741845_1021 (size=6437) 2024-11-23T22:49:26,618 INFO [M:0;dd96e8dc8369:36871 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.12 KB at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5911b946239e43928345f807ed267f6b 2024-11-23T22:49:26,642 DEBUG [M:0;dd96e8dc8369:36871 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/50870f8bd58049819b0cbf8ae998092e is 69, key is dd96e8dc8369,36415,1732402163807/rs:state/1732402164413/Put/seqid=0 2024-11-23T22:49:26,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741846_1022 (size=5294) 2024-11-23T22:49:26,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741846_1022 (size=5294) 2024-11-23T22:49:26,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41827 is added to blk_1073741846_1022 (size=5294) 2024-11-23T22:49:26,652 INFO [M:0;dd96e8dc8369:36871 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=195 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/50870f8bd58049819b0cbf8ae998092e 2024-11-23T22:49:26,662 DEBUG [M:0;dd96e8dc8369:36871 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a4afa770a87c4f58aa9298d380297190 as hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a4afa770a87c4f58aa9298d380297190 2024-11-23T22:49:26,673 INFO [M:0;dd96e8dc8369:36871 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a4afa770a87c4f58aa9298d380297190, entries=8, sequenceid=72, filesize=5.5 K 2024-11-23T22:49:26,674 DEBUG [M:0;dd96e8dc8369:36871 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5911b946239e43928345f807ed267f6b as hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5911b946239e43928345f807ed267f6b 2024-11-23T22:49:26,684 INFO [M:0;dd96e8dc8369:36871 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5911b946239e43928345f807ed267f6b, entries=8, sequenceid=72, filesize=6.3 K 2024-11-23T22:49:26,685 DEBUG [M:0;dd96e8dc8369:36871 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/50870f8bd58049819b0cbf8ae998092e as hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/50870f8bd58049819b0cbf8ae998092e 2024-11-23T22:49:26,693 INFO [M:0;dd96e8dc8369:36871 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42261/user/jenkins/test-data/af659b4e-c496-2d4b-00da-5e593b21b0da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/50870f8bd58049819b0cbf8ae998092e, entries=3, sequenceid=72, filesize=5.2 K 2024-11-23T22:49:26,694 INFO [M:0;dd96e8dc8369:36871 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.81 KB/27450, heapSize ~33.80 KB/34616, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 138ms, sequenceid=72, compaction requested=false 2024-11-23T22:49:26,696 INFO [M:0;dd96e8dc8369:36871 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:49:26,696 DEBUG [M:0;dd96e8dc8369:36871 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732402166556Disabling compacts and flushes for region at 1732402166556Disabling writes for close at 1732402166556Obtaining lock to block concurrent updates at 1732402166556Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732402166556Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=27450, getHeapSize=34856, getOffHeapSize=0, getCellsCount=85 at 1732402166557 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732402166557Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732402166558 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732402166576 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732402166576Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732402166592 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732402166609 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732402166609Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732402166626 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732402166642 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732402166642Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45c402e: reopening flushed file at 1732402166661 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f371451: reopening flushed file at 1732402166673 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69912b48: reopening flushed file at 1732402166684 (+11 ms)Finished flush of dataSize ~26.81 KB/27450, heapSize ~33.80 KB/34616, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 138ms, sequenceid=72, compaction requested=false at 1732402166695 (+11 ms)Writing region close event to WAL at 1732402166696 (+1 ms)Closed at 1732402166696 2024-11-23T22:49:26,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T22:49:26,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38213 is added to blk_1073741830_1006 (size=32653) 2024-11-23T22:49:26,703 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T22:49:26,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41827 is added to blk_1073741830_1006 (size=32653) 2024-11-23T22:49:26,703 INFO [M:0;dd96e8dc8369:36871 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T22:49:26,704 INFO [M:0;dd96e8dc8369:36871 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36871 2024-11-23T22:49:26,704 INFO [M:0;dd96e8dc8369:36871 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T22:49:26,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39807 is added to blk_1073741830_1006 (size=32653) 2024-11-23T22:49:26,811 INFO [M:0;dd96e8dc8369:36871 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T22:49:26,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:49:26,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36871-0x10169e32b2b0000, quorum=127.0.0.1:64210, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:49:26,818 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7c3d82b3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T22:49:26,819 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@190023f0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T22:49:26,819 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T22:49:26,819 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3c97cc8b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T22:49:26,820 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e74bc8b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/hadoop.log.dir/,STOPPED} 2024-11-23T22:49:26,823 WARN [BP-710832709-172.17.0.2-1732402160863 heartbeating to localhost/127.0.0.1:42261 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T22:49:26,823 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T22:49:26,823 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T22:49:26,823 WARN [BP-710832709-172.17.0.2-1732402160863 heartbeating to localhost/127.0.0.1:42261 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-710832709-172.17.0.2-1732402160863 (Datanode Uuid 4fafd9b6-edc3-47a5-8faa-c6465e36b7ec) service to localhost/127.0.0.1:42261 2024-11-23T22:49:26,824 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/cluster_3cba1636-64cb-f804-3a33-58a581002c60/data/data5/current/BP-710832709-172.17.0.2-1732402160863 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T22:49:26,824 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/cluster_3cba1636-64cb-f804-3a33-58a581002c60/data/data6/current/BP-710832709-172.17.0.2-1732402160863 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T22:49:26,825 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T22:49:26,837 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@25cc5f4d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T22:49:26,838 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6686fe53{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T22:49:26,838 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T22:49:26,838 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72785dee{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T22:49:26,838 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5c773fd1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/hadoop.log.dir/,STOPPED} 2024-11-23T22:49:26,840 WARN [BP-710832709-172.17.0.2-1732402160863 heartbeating to localhost/127.0.0.1:42261 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T22:49:26,840 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T22:49:26,840 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T22:49:26,840 WARN [BP-710832709-172.17.0.2-1732402160863 heartbeating to localhost/127.0.0.1:42261 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-710832709-172.17.0.2-1732402160863 (Datanode Uuid 7b537fac-16c8-4935-8cfd-e900380da330) service to localhost/127.0.0.1:42261 2024-11-23T22:49:26,840 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/cluster_3cba1636-64cb-f804-3a33-58a581002c60/data/data3/current/BP-710832709-172.17.0.2-1732402160863 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T22:49:26,841 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/cluster_3cba1636-64cb-f804-3a33-58a581002c60/data/data4/current/BP-710832709-172.17.0.2-1732402160863 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T22:49:26,841 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T22:49:26,849 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@11cbed31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T22:49:26,850 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11812ea4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T22:49:26,850 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T22:49:26,850 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@68e19264{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T22:49:26,850 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5e23bf16{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/hadoop.log.dir/,STOPPED} 2024-11-23T22:49:26,852 WARN [BP-710832709-172.17.0.2-1732402160863 heartbeating to localhost/127.0.0.1:42261 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T22:49:26,852 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T22:49:26,852 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T22:49:26,852 WARN [BP-710832709-172.17.0.2-1732402160863 heartbeating to localhost/127.0.0.1:42261 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-710832709-172.17.0.2-1732402160863 (Datanode Uuid d5ccd38d-3b80-4f7f-930e-4ad9e622e6cc) service to localhost/127.0.0.1:42261 2024-11-23T22:49:26,853 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/cluster_3cba1636-64cb-f804-3a33-58a581002c60/data/data1/current/BP-710832709-172.17.0.2-1732402160863 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T22:49:26,853 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/cluster_3cba1636-64cb-f804-3a33-58a581002c60/data/data2/current/BP-710832709-172.17.0.2-1732402160863 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T22:49:26,853 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T22:49:26,859 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@10db3f46{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T22:49:26,859 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43a7f4cb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T22:49:26,860 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T22:49:26,860 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4072566{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T22:49:26,860 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@17794d45{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/abf4c4a1-6544-5b1c-0bdf-8ae86926164c/hadoop.log.dir/,STOPPED} 2024-11-23T22:49:26,868 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T22:49:26,904 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-23T22:49:26,913 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestHBaseWalOnEC#testReadWrite[1] Thread=146 (was 87) - Thread LEAK? -, OpenFileDescriptor=518 (was 433) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=570 (was 558) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6720 (was 6916)