2023-02-12 21:02:50,285 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@31c88ec8
2023-02-12 21:02:50,300 main DEBUG Took 0.011963 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2023-02-12 21:02:50,300 main DEBUG PluginManager 'Core' found 128 plugins
2023-02-12 21:02:50,301 main DEBUG PluginManager 'Level' found 0 plugins
2023-02-12 21:02:50,302 main DEBUG PluginManager 'Lookup' found 16 plugins
2023-02-12 21:02:50,303 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2023-02-12 21:02:50,312 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2023-02-12 21:02:50,327 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2023-02-12 21:02:50,329 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2023-02-12 21:02:50,329 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2023-02-12 21:02:50,330 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2023-02-12 21:02:50,330 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2023-02-12 21:02:50,331 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2023-02-12 21:02:50,332 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2023-02-12 21:02:50,332 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2023-02-12 21:02:50,333 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2023-02-12 21:02:50,333 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2023-02-12 21:02:50,334 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2023-02-12 21:02:50,334 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2023-02-12 21:02:50,335 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2023-02-12 21:02:50,335 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2023-02-12 21:02:50,335 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2023-02-12 21:02:50,336 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2023-02-12 21:02:50,336 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2023-02-12 21:02:50,336 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2023-02-12 21:02:50,337 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2023-02-12 21:02:50,337 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2023-02-12 21:02:50,337 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2023-02-12 21:02:50,338 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2023-02-12 21:02:50,338 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2023-02-12 21:02:50,339 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2023-02-12 21:02:50,340 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2023-02-12 21:02:50,340 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2023-02-12 21:02:50,342 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2023-02-12 21:02:50,345 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2023-02-12 21:02:50,347 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2023-02-12 21:02:50,347 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2023-02-12 21:02:50,349 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2023-02-12 21:02:50,349 main DEBUG PluginManager 'Converter' found 45 plugins
2023-02-12 21:02:50,360 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2023-02-12 21:02:50,363 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2023-02-12 21:02:50,364 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2023-02-12 21:02:50,364 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2023-02-12 21:02:50,365 main DEBUG createAppenders(={Console})
2023-02-12 21:02:50,366 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@31c88ec8 initialized
2023-02-12 21:02:50,366 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@31c88ec8
2023-02-12 21:02:50,366 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@31c88ec8 OK.
2023-02-12 21:02:50,367 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2023-02-12 21:02:50,367 main DEBUG OutputStream closed
2023-02-12 21:02:50,367 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2023-02-12 21:02:50,367 main DEBUG Appender DefaultConsole-1 stopped with status true
2023-02-12 21:02:50,368 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@2cbb3d47 OK
2023-02-12 21:02:50,426 main DEBUG Registering MBean org.apache.logging.log4j2:type=3d4eac69
2023-02-12 21:02:50,428 main DEBUG Registering MBean org.apache.logging.log4j2:type=3d4eac69,component=StatusLogger
2023-02-12 21:02:50,429 main DEBUG Registering MBean org.apache.logging.log4j2:type=3d4eac69,component=ContextSelector
2023-02-12 21:02:50,430 main DEBUG Registering MBean org.apache.logging.log4j2:type=3d4eac69,component=Loggers,name=
2023-02-12 21:02:50,431 main DEBUG Registering MBean org.apache.logging.log4j2:type=3d4eac69,component=Loggers,name=org.apache.directory
2023-02-12 21:02:50,431 main DEBUG Registering MBean org.apache.logging.log4j2:type=3d4eac69,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2023-02-12 21:02:50,432 main DEBUG Registering MBean org.apache.logging.log4j2:type=3d4eac69,component=Loggers,name=org.apache.zookeeper
2023-02-12 21:02:50,432 main DEBUG Registering MBean org.apache.logging.log4j2:type=3d4eac69,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2023-02-12 21:02:50,433 main DEBUG Registering MBean org.apache.logging.log4j2:type=3d4eac69,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2023-02-12 21:02:50,433 main DEBUG Registering MBean org.apache.logging.log4j2:type=3d4eac69,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2023-02-12 21:02:50,434 main DEBUG Registering MBean org.apache.logging.log4j2:type=3d4eac69,component=Loggers,name=org.apache.hadoop.hbase
2023-02-12 21:02:50,434 main DEBUG Registering MBean org.apache.logging.log4j2:type=3d4eac69,component=Loggers,name=org.apache.hadoop
2023-02-12 21:02:50,435 main DEBUG Registering MBean org.apache.logging.log4j2:type=3d4eac69,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2023-02-12 21:02:50,435 main DEBUG Registering MBean org.apache.logging.log4j2:type=3d4eac69,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2023-02-12 21:02:50,436 main DEBUG Registering MBean org.apache.logging.log4j2:type=3d4eac69,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2023-02-12 21:02:50,436 main DEBUG Registering MBean org.apache.logging.log4j2:type=3d4eac69,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2023-02-12 21:02:50,436 main DEBUG Registering MBean org.apache.logging.log4j2:type=3d4eac69,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2023-02-12 21:02:50,438 main DEBUG Registering MBean org.apache.logging.log4j2:type=3d4eac69,component=Appenders,name=Console
2023-02-12 21:02:50,440 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps.
2023-02-12 21:02:50,440 main DEBUG Reconfiguration complete for context[name=3d4eac69] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@71e9ddb4) with optional ClassLoader: null
2023-02-12 21:02:50,440 main DEBUG Shutdown hook enabled. Registering a new one.
2023-02-12 21:02:50,445 main DEBUG LoggerContext[name=3d4eac69, org.apache.logging.log4j.core.LoggerContext@71e9ddb4] started OK.
2023-02-12T21:02:50,464 INFO  [main] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure timeout: 13 mins
2023-02-12T21:02:50,703 INFO  [Time-limited test] hbase.ResourceChecker(147): before: procedure2.TestForceUpdateProcedure#testProcedureStuck Thread=8, OpenFileDescriptor=110, MaxFileDescriptor=60000, SystemLoadAverage=275, ProcessCount=167, AvailableMemoryMB=13163
2023-02-12T21:02:50,826 WARN  [Time-limited test] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2023-02-12T21:02:50,954 INFO  [Time-limited test] procedure2.ProcedureExecutor(564): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2023-02-12T21:02:50,958 DEBUG [Time-limited test] wal.WALProcedureStore(398): Starting WAL Procedure Store lease recovery
2023-02-12T21:02:50,989 DEBUG [Time-limited test] util.CommonFSUtils$DfsBuilderUtility(744): org.apache.hadoop.hdfs.DistributedFileSystem$HdfsDataOutputStreamBuilder not available, will not set replicate when creating output stream
2023-02-12T21:02:51,016 INFO  [Time-limited test] wal.WALProcedureStore(1126): Rolled new Procedure Store WAL, id=1
2023-02-12T21:02:51,018 DEBUG [Time-limited test] wal.WALProcedureStore(432): Lease acquired for flushLogId=1
2023-02-12T21:02:51,018 INFO  [Time-limited test] procedure2.ProcedureExecutor(584): Recovered WALProcedureStore lease in 60 msec
2023-02-12T21:02:51,021 DEBUG [Time-limited test] wal.WALProcedureStore(450): No state logs to replay.
2023-02-12T21:02:51,021 INFO  [Time-limited test] procedure2.ProcedureExecutor(598): Loaded WALProcedureStore in 2 msec
2023-02-12T21:02:51,025 INFO  [Time-limited test] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=1000, timestamp=1676235772024
2023-02-12T21:02:51,062 DEBUG [Time-limited test] procedure2.ProcedureExecutor(1031): Stored pid=1, state=RUNNABLE; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$ParentProcedure
2023-02-12T21:02:51,098 INFO  [PEWorker-1] procedure2.ProcedureExecutor(1683): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE; org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility$NoopProcedure}, {pid=3, ppid=1, state=RUNNABLE; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$WaitingProcedure}]
2023-02-12T21:02:51,100 INFO  [PEWorker-1] procedure2.TimeoutExecutorThread(81): ADDED pid=3, ppid=1, state=WAITING_TIMEOUT, locked=true; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$WaitingProcedure; timeout=2147483647, timestamp=1678383254747
2023-02-12T21:02:51,102 INFO  [PEWorker-1] procedure2.ProcedureExecutor(1412): Finished pid=2, ppid=1, state=SUCCESS; org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility$NoopProcedure in 7 msec
2023-02-12T21:02:51,103 INFO  [Time-limited test] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2023-02-12T21:02:51,104 DEBUG [Time-limited test] procedure2.ProcedureExecutor(1031): Stored pid=4, state=RUNNABLE; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$ExchangeProcedure
2023-02-12T21:02:51,130 INFO  [Time-limited test] wal.WALProcedureStore(1126): Rolled new Procedure Store WAL, id=2
2023-02-12T21:02:52,138 INFO  [Time-limited test] wal.WALProcedureStore(1126): Rolled new Procedure Store WAL, id=3
2023-02-12T21:02:53,142 INFO  [Time-limited test] wal.WALProcedureStore(1126): Rolled new Procedure Store WAL, id=4
2023-02-12T21:02:54,146 INFO  [Time-limited test] wal.WALProcedureStore(1126): Rolled new Procedure Store WAL, id=5
2023-02-12T21:02:55,152 WARN  [Time-limited test] wal.WALProcedureStore(1111): procedure WALs count=6 above the warning threshold 5. check running procedures to see if something is stuck.
2023-02-12T21:02:55,162 INFO  [Time-limited test] wal.WALProcedureStore(1126): Rolled new Procedure Store WAL, id=6
2023-02-12T21:02:55,162 DEBUG [Force-Update-PEWorker-0] procedure2.ProcedureExecutor(290): Force update procedure pid=1, state=WAITING; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$ParentProcedure
2023-02-12T21:02:55,164 DEBUG [Force-Update-PEWorker-0] procedure2.ProcedureExecutor(290): Force update procedure pid=2, ppid=1, state=SUCCESS; org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility$NoopProcedure
2023-02-12T21:02:55,165 INFO  [Time-limited test] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2023-02-12T21:02:55,165 DEBUG [Force-Update-PEWorker-0] procedure2.ProcedureExecutor(290): Force update procedure pid=3, ppid=1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$WaitingProcedure
2023-02-12T21:02:55,165 INFO  [WALProcedureStoreSyncThread] wal.WALProcedureStore(1213): Remove all state logs with ID less than 5, since all the active procedures are in the latest log
2023-02-12T21:02:55,165 INFO  [PEWorker-1] procedure2.ProcedureExecutor(1412): Finished pid=4, state=SUCCESS; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$ExchangeProcedure in 4.0610 sec
2023-02-12T21:02:55,166 DEBUG [WALProcedureStoreSyncThread] wal.WALProcedureStore(1235): Removed log=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000001.log, activeLogs=[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000002.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000003.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000004.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000005.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000006.log]
2023-02-12T21:02:55,167 DEBUG [WALProcedureStoreSyncThread] wal.WALProcedureStore(1235): Removed log=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000002.log, activeLogs=[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000003.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000004.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000005.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000006.log]
2023-02-12T21:02:55,168 DEBUG [WALProcedureStoreSyncThread] wal.WALProcedureStore(1235): Removed log=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000003.log, activeLogs=[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000004.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000005.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000006.log]
2023-02-12T21:02:55,169 DEBUG [WALProcedureStoreSyncThread] wal.WALProcedureStore(1235): Removed log=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000004.log, activeLogs=[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000005.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000006.log]
2023-02-12T21:02:55,169 DEBUG [WALProcedureStoreSyncThread] wal.WALProcedureStore(1235): Removed log=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000005.log, activeLogs=[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000006.log]
2023-02-12T21:02:55,267 INFO  [Time-limited test] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2023-02-12T21:02:55,267 INFO  [Time-limited test] procedure2.ProcedureExecutor(631): Stopping
2023-02-12T21:02:55,268 INFO  [Time-limited test] wal.WALProcedureStore(327): Stopping the WAL Procedure Store, isAbort=false
2023-02-12T21:02:55,270 INFO  [Time-limited test] procedure2.ProcedureExecutor(564): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2023-02-12T21:02:55,270 DEBUG [Time-limited test] wal.WALProcedureStore(398): Starting WAL Procedure Store lease recovery
2023-02-12T21:02:55,271 DEBUG [Time-limited test] wal.WALProcedureStore(1365): Opening Pv2 DeprecatedRawLocalFileStatus{path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000006.log; isDirectory=false; length=588; replication=1; blocksize=33554432; modification_time=1676235775265; access_time=1676235775145; owner=; group=; permission=rw-rw-rw-; isSymlink=false}
2023-02-12T21:02:55,286 INFO  [Time-limited test] wal.WALProcedureStore(1126): Rolled new Procedure Store WAL, id=7
2023-02-12T21:02:55,287 DEBUG [Time-limited test] wal.WALProcedureStore(432): Lease acquired for flushLogId=7
2023-02-12T21:02:55,287 INFO  [Time-limited test] procedure2.ProcedureExecutor(584): Recovered WALProcedureStore lease in 17 msec
2023-02-12T21:02:55,293 INFO  [Time-limited test] wal.ProcedureWALFormatReader(124): Read 6 entries in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testProcedureStuck/pv2-00000000000000000006.log
2023-02-12T21:02:55,298 DEBUG [Time-limited test] store.ProcedureTree(114): Procedure Procedure(pid=1, ppid=-1, class=org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$ParentProcedure) stack ids=[0]
2023-02-12T21:02:55,300 DEBUG [Time-limited test] store.ProcedureTree(114): Procedure Procedure(pid=2, ppid=1, class=org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility$NoopProcedure) stack ids=[2]
2023-02-12T21:02:55,300 DEBUG [Time-limited test] store.ProcedureTree(114): Procedure Procedure(pid=3, ppid=1, class=org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$WaitingProcedure) stack ids=[1]
2023-02-12T21:02:55,303 DEBUG [Time-limited test] procedure2.ProcedureExecutor(413): Completed pid=4, state=SUCCESS; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$ExchangeProcedure
2023-02-12T21:02:55,303 DEBUG [Time-limited test] procedure2.ProcedureExecutor(470): Loading pid=1, state=WAITING; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$ParentProcedure
2023-02-12T21:02:55,303 DEBUG [Time-limited test] procedure2.ProcedureExecutor(470): Loading pid=2, ppid=1, state=SUCCESS; org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility$NoopProcedure
2023-02-12T21:02:55,303 DEBUG [Time-limited test] procedure2.ProcedureExecutor(470): Loading pid=3, ppid=1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$WaitingProcedure
2023-02-12T21:02:55,305 DEBUG [Time-limited test] procedure2.Procedure(946): pid=1, state=WAITING; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$ParentProcedure didn't hold the lock before restarting, skip acquiring lock.
2023-02-12T21:02:55,305 DEBUG [Time-limited test] procedure2.Procedure(946): pid=2, ppid=1, state=SUCCESS; org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility$NoopProcedure didn't hold the lock before restarting, skip acquiring lock.
2023-02-12T21:02:55,305 DEBUG [Time-limited test] procedure2.Procedure(946): pid=3, ppid=1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$WaitingProcedure didn't hold the lock before restarting, skip acquiring lock.
2023-02-12T21:02:55,306 INFO  [Time-limited test] procedure2.TimeoutExecutorThread(81): ADDED pid=3, ppid=1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$WaitingProcedure; timeout=2147483647, timestamp=1678383254747
2023-02-12T21:02:55,308 INFO  [Time-limited test] procedure2.ProcedureExecutor(598): Loaded WALProcedureStore in 19 msec
2023-02-12T21:02:55,308 INFO  [Time-limited test] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=1000, timestamp=1676235776308
2023-02-12T21:02:55,309 INFO  [Time-limited test] procedure2.ProcedureExecutor(631): Stopping
2023-02-12T21:02:55,310 INFO  [Time-limited test] wal.WALProcedureStore(327): Stopping the WAL Procedure Store, isAbort=false
2023-02-12T21:02:55,323 INFO  [Time-limited test] hbase.ResourceChecker(175): after: procedure2.TestForceUpdateProcedure#testProcedureStuck Thread=10 (was 8)
Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.lang.Object.wait(Native Method)
	java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144)
	java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165)
	org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3712)
	java.lang.Thread.run(Thread.java:750)

Potentially hanging thread: Force-Update-PEWorker-0
	sun.misc.Unsafe.park(Native Method)
	java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
	java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
	java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
	java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	java.lang.Thread.run(Thread.java:750)
 - Thread LEAK? -, OpenFileDescriptor=110 (was 110), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=275 (was 275), ProcessCount=167 (was 167), AvailableMemoryMB=13037 (was 13163)
2023-02-12T21:02:55,333 INFO  [Time-limited test] hbase.ResourceChecker(147): before: procedure2.TestForceUpdateProcedure#testCompletedProcedure Thread=10, OpenFileDescriptor=110, MaxFileDescriptor=60000, SystemLoadAverage=275, ProcessCount=167, AvailableMemoryMB=13037
2023-02-12T21:02:55,336 INFO  [Time-limited test] procedure2.ProcedureExecutor(564): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2023-02-12T21:02:55,336 DEBUG [Time-limited test] wal.WALProcedureStore(398): Starting WAL Procedure Store lease recovery
2023-02-12T21:02:55,340 INFO  [Time-limited test] wal.WALProcedureStore(1126): Rolled new Procedure Store WAL, id=1
2023-02-12T21:02:55,341 DEBUG [Time-limited test] wal.WALProcedureStore(432): Lease acquired for flushLogId=1
2023-02-12T21:02:55,341 INFO  [Time-limited test] procedure2.ProcedureExecutor(584): Recovered WALProcedureStore lease in 4 msec
2023-02-12T21:02:55,341 DEBUG [Time-limited test] wal.WALProcedureStore(450): No state logs to replay.
2023-02-12T21:02:55,341 INFO  [Time-limited test] procedure2.ProcedureExecutor(598): Loaded WALProcedureStore in 0 msec
2023-02-12T21:02:55,347 INFO  [Time-limited test] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=1000, timestamp=1676235776346
2023-02-12T21:02:55,347 DEBUG [Time-limited test] procedure2.ProcedureExecutor(1031): Stored pid=1, state=RUNNABLE; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$ExchangeProcedure
2023-02-12T21:02:55,349 INFO  [Time-limited test] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2023-02-12T21:02:55,349 INFO  [PEWorker-1] procedure2.ProcedureExecutor(1412): Finished pid=1, state=SUCCESS; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$ExchangeProcedure in 1 msec
2023-02-12T21:02:55,452 INFO  [Time-limited test] wal.WALProcedureStore(1126): Rolled new Procedure Store WAL, id=2
2023-02-12T21:02:55,454 DEBUG [Time-limited test] procedure2.ProcedureExecutor(1031): Stored pid=2, state=RUNNABLE; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$NoopNoAckProcedure
2023-02-12T21:02:55,455 INFO  [Time-limited test] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2023-02-12T21:02:55,456 INFO  [PEWorker-1] procedure2.ProcedureExecutor(1412): Finished pid=2, state=SUCCESS; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$NoopNoAckProcedure in 3 msec
2023-02-12T21:02:55,558 INFO  [Time-limited test] wal.WALProcedureStore(1126): Rolled new Procedure Store WAL, id=3
2023-02-12T21:02:55,560 DEBUG [Time-limited test] procedure2.ProcedureExecutor(1031): Stored pid=3, state=RUNNABLE; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$NoopNoAckProcedure
2023-02-12T21:02:55,561 INFO  [Time-limited test] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2023-02-12T21:02:55,561 INFO  [PEWorker-1] procedure2.ProcedureExecutor(1412): Finished pid=3, state=SUCCESS; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$NoopNoAckProcedure in 2 msec
2023-02-12T21:02:55,664 INFO  [Time-limited test] wal.WALProcedureStore(1126): Rolled new Procedure Store WAL, id=4
2023-02-12T21:02:55,665 DEBUG [Time-limited test] procedure2.ProcedureExecutor(1031): Stored pid=4, state=RUNNABLE; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$NoopNoAckProcedure
2023-02-12T21:02:55,666 INFO  [Time-limited test] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2023-02-12T21:02:55,666 INFO  [PEWorker-1] procedure2.ProcedureExecutor(1412): Finished pid=4, state=SUCCESS; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$NoopNoAckProcedure in 2 msec
2023-02-12T21:02:55,769 INFO  [Time-limited test] wal.WALProcedureStore(1126): Rolled new Procedure Store WAL, id=5
2023-02-12T21:02:55,771 DEBUG [Time-limited test] procedure2.ProcedureExecutor(1031): Stored pid=5, state=RUNNABLE; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$NoopNoAckProcedure
2023-02-12T21:02:55,772 INFO  [Time-limited test] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2023-02-12T21:02:55,773 INFO  [PEWorker-1] procedure2.ProcedureExecutor(1412): Finished pid=5, state=SUCCESS; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$NoopNoAckProcedure in 1 msec
2023-02-12T21:02:55,873 INFO  [Time-limited test] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2023-02-12T21:02:56,376 WARN  [Time-limited test] wal.WALProcedureStore(1111): procedure WALs count=6 above the warning threshold 5. check running procedures to see if something is stuck.
2023-02-12T21:02:56,377 INFO  [Time-limited test] wal.WALProcedureStore(1126): Rolled new Procedure Store WAL, id=6
2023-02-12T21:02:56,377 DEBUG [Force-Update-PEWorker-0] procedure2.ProcedureExecutor(290): Force update procedure pid=1, state=SUCCESS; org.apache.hadoop.hbase.procedure2.TestForceUpdateProcedure$ExchangeProcedure
2023-02-12T21:02:56,379 INFO  [WALProcedureStoreSyncThread] wal.WALProcedureStore(1213): Remove all state logs with ID less than 5, since all the active procedures are in the latest log
2023-02-12T21:02:56,380 INFO  [Time-limited test] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2023-02-12T21:02:56,380 DEBUG [WALProcedureStoreSyncThread] wal.WALProcedureStore(1235): Removed log=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000001.log, activeLogs=[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000002.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000003.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000004.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000005.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000006.log]
2023-02-12T21:02:56,381 DEBUG [WALProcedureStoreSyncThread] wal.WALProcedureStore(1235): Removed log=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000002.log, activeLogs=[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000003.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000004.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000005.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000006.log]
2023-02-12T21:02:56,382 DEBUG [WALProcedureStoreSyncThread] wal.WALProcedureStore(1235): Removed log=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000003.log, activeLogs=[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000004.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000005.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000006.log]
2023-02-12T21:02:56,382 DEBUG [WALProcedureStoreSyncThread] wal.WALProcedureStore(1235): Removed log=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000004.log, activeLogs=[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000005.log, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000006.log]
2023-02-12T21:02:56,383 DEBUG [WALProcedureStoreSyncThread] wal.WALProcedureStore(1235): Removed log=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000005.log, activeLogs=[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/test-data/36634d75-c37b-914b-1413-5d02f9503a42/testCompletedProcedure/pv2-00000000000000000006.log]
2023-02-12T21:02:56,386 INFO  [Time-limited test] procedure2.ProcedureExecutor(631): Stopping
2023-02-12T21:02:56,388 INFO  [Time-limited test] wal.WALProcedureStore(327): Stopping the WAL Procedure Store, isAbort=false
2023-02-12T21:02:56,398 INFO  [Time-limited test] hbase.ResourceChecker(175): after: procedure2.TestForceUpdateProcedure#testCompletedProcedure Thread=11 (was 10) - Thread LEAK? -, OpenFileDescriptor=110 (was 110), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=285 (was 275) - SystemLoadAverage LEAK? -, ProcessCount=167 (was 167), AvailableMemoryMB=13034 (was 13037)