public class DefaultExecutionGraph extends Object implements ExecutionGraph, InternalExecutionGraphAccessor
ExecutionGraph
.Constructor and Description |
---|
DefaultExecutionGraph(JobInformation jobInformation,
ScheduledExecutorService futureExecutor,
Executor ioExecutor,
Time rpcTimeout,
int executionHistorySizeLimit,
ClassLoader userClassLoader,
BlobWriter blobWriter,
PartitionGroupReleaseStrategy.Factory partitionGroupReleaseStrategyFactory,
ShuffleMaster<?> shuffleMaster,
JobMasterPartitionTracker partitionTracker,
ExecutionDeploymentListener executionDeploymentListener,
ExecutionStateUpdateListener executionStateUpdateListener,
long initializationTimestamp,
VertexAttemptNumberStore initialAttemptCounts,
VertexParallelismStore vertexParallelismStore,
boolean isDynamic,
ExecutionJobVertex.Factory executionJobVertexFactory,
List<JobStatusHook> jobStatusHooks,
MarkPartitionFinishedStrategy markPartitionFinishedStrategy,
TaskDeploymentDescriptorFactory taskDeploymentDescriptorFactory) |
Modifier and Type | Method and Description |
---|---|
Map<String,OptionalFailure<Accumulator<?,?>>> |
aggregateUserAccumulators()
Merges all accumulator results from the tasks previously executed in the Executions.
|
void |
attachJobGraph(List<JobVertex> verticesToAttach,
JobManagerJobMetricGroup jobManagerJobMetricGroup) |
void |
cancel() |
protected FutureUtils.ConjunctFuture<Void> |
cancelVerticesAsync() |
void |
deleteBlobs(List<PermanentBlobKey> blobKeys) |
void |
deregisterExecution(Execution exec) |
void |
enableCheckpointing(CheckpointCoordinatorConfiguration chkConfig,
List<MasterTriggerRestoreHook<?>> masterHooks,
CheckpointIDCounter checkpointIDCounter,
CompletedCheckpointStore checkpointStore,
StateBackend checkpointStateBackend,
CheckpointStorage checkpointStorage,
CheckpointStatsTracker statsTracker,
CheckpointsCleaner checkpointsCleaner,
String changelogStorageName) |
void |
failGlobal(Throwable t)
Fails the execution graph globally.
|
void |
failJob(Throwable cause,
long timestamp) |
Optional<AccessExecution> |
findExecution(ExecutionAttemptID attemptId) |
Optional<String> |
findVertexWithAttempt(ExecutionAttemptID attemptId) |
StringifiedAccumulatorResult[] |
getAccumulatorResultsStringified()
Returns the a stringified version of the user-defined accumulators.
|
Map<String,SerializedValue<OptionalFailure<Object>>> |
getAccumulatorsSerialized()
Gets a serialized accumulator map.
|
Iterable<ExecutionVertex> |
getAllExecutionVertices()
Returns an iterable containing all execution vertices for this execution graph.
|
Map<IntermediateDataSetID,IntermediateResult> |
getAllIntermediateResults() |
Map<JobVertexID,ExecutionJobVertex> |
getAllVertices()
Returns a map containing all job vertices for this execution graph.
|
ArchivedExecutionConfig |
getArchivedExecutionConfig()
Returns the serializable
ArchivedExecutionConfig . |
BlobWriter |
getBlobWriter() |
Optional<String> |
getChangelogStorageName()
Returns the changelog storage name for this ExecutionGraph.
|
CheckpointCoordinator |
getCheckpointCoordinator() |
CheckpointCoordinatorConfiguration |
getCheckpointCoordinatorConfiguration()
Returns the
CheckpointCoordinatorConfiguration or null if checkpointing
is disabled. |
CheckpointStatsSnapshot |
getCheckpointStatsSnapshot()
Returns a snapshot of the checkpoint statistics or
null if checkpointing is
disabled. |
Optional<String> |
getCheckpointStorageName()
Returns the checkpoint storage name for this ExecutionGraph.
|
List<ShuffleDescriptor> |
getClusterPartitionShuffleDescriptors(IntermediateDataSetID intermediateDataSetID)
Get the shuffle descriptors of the cluster partitions ordered by partition number.
|
EdgeManager |
getEdgeManager() |
ExecutionDeploymentListener |
getExecutionDeploymentListener() |
ExecutionGraphID |
getExecutionGraphID() |
ExecutionVertex |
getExecutionVertexOrThrow(ExecutionVertexID id) |
Throwable |
getFailureCause() |
ErrorInfo |
getFailureInfo()
Returns the exception that caused the job to fail.
|
Executor |
getFutureExecutor()
Returns the ExecutionContext associated with this ExecutionGraph.
|
Configuration |
getJobConfiguration() |
JobID |
getJobID()
Returns the
JobID for this execution graph. |
ComponentMainThreadExecutor |
getJobMasterMainThreadExecutor() |
String |
getJobName()
Returns the job name for the execution graph.
|
ExecutionJobVertex |
getJobVertex(JobVertexID id)
Returns the job vertex for the given
JobVertexID . |
JobVertexInputInfo |
getJobVertexInputInfo(JobVertexID jobVertexId,
IntermediateDataSetID resultId)
Get the input info of a certain input of a certain job vertex.
|
String |
getJsonPlan()
Returns the job plan as a JSON string.
|
KvStateLocationRegistry |
getKvStateLocationRegistry() |
MarkPartitionFinishedStrategy |
getMarkPartitionFinishedStrategy() |
long |
getNumberOfRestarts()
Gets the number of restarts, including full restarts and fine grained restarts.
|
int |
getNumFinishedVertices() |
PartitionGroupReleaseStrategy |
getPartitionGroupReleaseStrategy() |
JobMasterPartitionTracker |
getPartitionTracker() |
Map<ExecutionAttemptID,Execution> |
getRegisteredExecutions() |
ResultPartitionAvailabilityChecker |
getResultPartitionAvailabilityChecker() |
IntermediateResultPartition |
getResultPartitionOrThrow(IntermediateResultPartitionID id)
Gets the intermediate result partition by the given partition ID, or throw an exception if
the partition is not found.
|
SchedulingTopology |
getSchedulingTopology() |
ShuffleMaster<?> |
getShuffleMaster() |
JobStatus |
getState()
Returns the current
JobStatus for this execution graph. |
Optional<String> |
getStateBackendName()
Returns the state backend name for this ExecutionGraph.
|
long |
getStatusTimestamp(JobStatus status)
Returns the timestamp for the given
JobStatus . |
TaskDeploymentDescriptorFactory |
getTaskDeploymentDescriptorFactory() |
CompletableFuture<JobStatus> |
getTerminationFuture()
Returns the termination future of this
ExecutionGraph . |
ClassLoader |
getUserClassLoader() |
Iterable<ExecutionJobVertex> |
getVerticesTopologically()
Returns an iterable containing all job vertices for this execution graph in the order they
were created.
|
void |
incrementRestarts() |
void |
initFailureCause(Throwable t,
long timestamp) |
void |
initializeJobVertex(ExecutionJobVertex ejv,
long createTimestamp,
Map<IntermediateDataSetID,JobVertexInputInfo> jobVertexInputInfos,
JobManagerJobMetricGroup jobManagerJobMetricGroup)
Initialize the given execution job vertex, mainly includes creating execution vertices
according to the parallelism, and connecting to the predecessors.
|
TernaryBoolean |
isChangelogStateBackendEnabled()
Returns whether the state changelog is enabled for this ExecutionGraph.
|
boolean |
isDynamic() |
boolean |
isStoppable()
Returns whether the job for this execution graph is stoppable.
|
void |
jobVertexFinished()
Called whenever a job vertex reaches state FINISHED (completed successfully).
|
void |
jobVertexUnFinished() |
void |
notifyExecutionChange(Execution execution,
ExecutionState previousState,
ExecutionState newExecutionState) |
void |
notifyNewlyInitializedJobVertices(List<ExecutionJobVertex> vertices)
Notify that some job vertices have been newly initialized, execution graph will try to update
scheduling topology.
|
void |
notifySchedulerNgAboutInternalTaskFailure(ExecutionAttemptID attemptId,
Throwable t,
boolean cancelTask,
boolean releasePartitions) |
void |
registerExecution(Execution exec) |
void |
registerJobStatusListener(JobStatusListener listener) |
void |
setInternalTaskFailuresListener(InternalFailuresListener internalTaskFailuresListener) |
void |
setJsonPlan(String jsonPlan) |
void |
start(ComponentMainThreadExecutor jobMasterMainThreadExecutor) |
void |
suspend(Throwable suspensionCause)
Suspends the current ExecutionGraph.
|
boolean |
transitionState(JobStatus current,
JobStatus newState) |
void |
transitionToRunning() |
void |
updateAccumulators(AccumulatorSnapshot accumulatorSnapshot)
Updates the accumulators during the runtime of a job.
|
boolean |
updateState(TaskExecutionStateTransition state)
Updates the state of one of the ExecutionVertex's Execution attempts.
|
JobStatus |
waitUntilTerminal() |
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
initializeJobVertex
public DefaultExecutionGraph(JobInformation jobInformation, ScheduledExecutorService futureExecutor, Executor ioExecutor, Time rpcTimeout, int executionHistorySizeLimit, ClassLoader userClassLoader, BlobWriter blobWriter, PartitionGroupReleaseStrategy.Factory partitionGroupReleaseStrategyFactory, ShuffleMaster<?> shuffleMaster, JobMasterPartitionTracker partitionTracker, ExecutionDeploymentListener executionDeploymentListener, ExecutionStateUpdateListener executionStateUpdateListener, long initializationTimestamp, VertexAttemptNumberStore initialAttemptCounts, VertexParallelismStore vertexParallelismStore, boolean isDynamic, ExecutionJobVertex.Factory executionJobVertexFactory, List<JobStatusHook> jobStatusHooks, MarkPartitionFinishedStrategy markPartitionFinishedStrategy, TaskDeploymentDescriptorFactory taskDeploymentDescriptorFactory)
public void start(@Nonnull ComponentMainThreadExecutor jobMasterMainThreadExecutor)
start
in interface ExecutionGraph
public SchedulingTopology getSchedulingTopology()
getSchedulingTopology
in interface ExecutionGraph
@Nonnull public ComponentMainThreadExecutor getJobMasterMainThreadExecutor()
getJobMasterMainThreadExecutor
in interface ExecutionGraph
getJobMasterMainThreadExecutor
in interface InternalExecutionGraphAccessor
public TernaryBoolean isChangelogStateBackendEnabled()
AccessExecutionGraph
isChangelogStateBackendEnabled
in interface AccessExecutionGraph
public Optional<String> getStateBackendName()
AccessExecutionGraph
getStateBackendName
in interface AccessExecutionGraph
public Optional<String> getCheckpointStorageName()
AccessExecutionGraph
getCheckpointStorageName
in interface AccessExecutionGraph
public Optional<String> getChangelogStorageName()
AccessExecutionGraph
getChangelogStorageName
in interface AccessExecutionGraph
public void enableCheckpointing(CheckpointCoordinatorConfiguration chkConfig, List<MasterTriggerRestoreHook<?>> masterHooks, CheckpointIDCounter checkpointIDCounter, CompletedCheckpointStore checkpointStore, StateBackend checkpointStateBackend, CheckpointStorage checkpointStorage, CheckpointStatsTracker statsTracker, CheckpointsCleaner checkpointsCleaner, String changelogStorageName)
enableCheckpointing
in interface ExecutionGraph
@Nullable public CheckpointCoordinator getCheckpointCoordinator()
getCheckpointCoordinator
in interface ExecutionGraph
public KvStateLocationRegistry getKvStateLocationRegistry()
getKvStateLocationRegistry
in interface ExecutionGraph
public CheckpointCoordinatorConfiguration getCheckpointCoordinatorConfiguration()
AccessExecutionGraph
CheckpointCoordinatorConfiguration
or null
if checkpointing
is disabled.getCheckpointCoordinatorConfiguration
in interface AccessExecutionGraph
public CheckpointStatsSnapshot getCheckpointStatsSnapshot()
AccessExecutionGraph
null
if checkpointing is
disabled.getCheckpointStatsSnapshot
in interface AccessExecutionGraph
public void setJsonPlan(String jsonPlan)
setJsonPlan
in interface ExecutionGraph
public String getJsonPlan()
AccessExecutionGraph
getJsonPlan
in interface AccessExecutionGraph
public JobID getJobID()
AccessExecutionGraph
JobID
for this execution graph.getJobID
in interface AccessExecutionGraph
getJobID
in interface InternalExecutionGraphAccessor
public String getJobName()
AccessExecutionGraph
getJobName
in interface AccessExecutionGraph
public boolean isStoppable()
AccessExecutionGraph
isStoppable
in interface AccessExecutionGraph
public Configuration getJobConfiguration()
getJobConfiguration
in interface ExecutionGraph
public ClassLoader getUserClassLoader()
getUserClassLoader
in interface InternalExecutionGraphAccessor
public JobStatus getState()
AccessExecutionGraph
JobStatus
for this execution graph.getState
in interface AccessExecutionGraph
getState
in interface JobStatusProvider
public Throwable getFailureCause()
getFailureCause
in interface ExecutionGraph
public ErrorInfo getFailureInfo()
AccessExecutionGraph
getFailureInfo
in interface AccessExecutionGraph
public long getNumberOfRestarts()
ExecutionGraph
getNumberOfRestarts
in interface ExecutionGraph
public int getNumFinishedVertices()
getNumFinishedVertices
in interface ExecutionGraph
public ExecutionJobVertex getJobVertex(JobVertexID id)
AccessExecutionGraph
JobVertexID
.getJobVertex
in interface AccessExecutionGraph
getJobVertex
in interface ExecutionGraph
getJobVertex
in interface InternalExecutionGraphAccessor
id
- id of job vertex to be returnednull
public Map<JobVertexID,ExecutionJobVertex> getAllVertices()
AccessExecutionGraph
getAllVertices
in interface AccessExecutionGraph
getAllVertices
in interface ExecutionGraph
public Iterable<ExecutionJobVertex> getVerticesTopologically()
AccessExecutionGraph
getVerticesTopologically
in interface AccessExecutionGraph
getVerticesTopologically
in interface ExecutionGraph
public Map<IntermediateDataSetID,IntermediateResult> getAllIntermediateResults()
getAllIntermediateResults
in interface ExecutionGraph
public Iterable<ExecutionVertex> getAllExecutionVertices()
AccessExecutionGraph
getAllExecutionVertices
in interface AccessExecutionGraph
getAllExecutionVertices
in interface ExecutionGraph
public EdgeManager getEdgeManager()
getEdgeManager
in interface InternalExecutionGraphAccessor
public ExecutionVertex getExecutionVertexOrThrow(ExecutionVertexID id)
getExecutionVertexOrThrow
in interface InternalExecutionGraphAccessor
public IntermediateResultPartition getResultPartitionOrThrow(IntermediateResultPartitionID id)
ExecutionGraph
getResultPartitionOrThrow
in interface ExecutionGraph
getResultPartitionOrThrow
in interface InternalExecutionGraphAccessor
id
- of the intermediate result partitionpublic long getStatusTimestamp(JobStatus status)
AccessExecutionGraph
JobStatus
.getStatusTimestamp
in interface AccessExecutionGraph
getStatusTimestamp
in interface JobStatusProvider
status
- status for which the timestamp should be returnedpublic final BlobWriter getBlobWriter()
getBlobWriter
in interface InternalExecutionGraphAccessor
public Executor getFutureExecutor()
InternalExecutionGraphAccessor
getFutureExecutor
in interface InternalExecutionGraphAccessor
public Map<String,OptionalFailure<Accumulator<?,?>>> aggregateUserAccumulators()
ExecutionGraph
aggregateUserAccumulators
in interface ExecutionGraph
public Map<String,SerializedValue<OptionalFailure<Object>>> getAccumulatorsSerialized()
getAccumulatorsSerialized
in interface AccessExecutionGraph
public StringifiedAccumulatorResult[] getAccumulatorResultsStringified()
getAccumulatorResultsStringified
in interface AccessExecutionGraph
public void setInternalTaskFailuresListener(InternalFailuresListener internalTaskFailuresListener)
setInternalTaskFailuresListener
in interface ExecutionGraph
public void notifyNewlyInitializedJobVertices(List<ExecutionJobVertex> vertices)
ExecutionGraph
notifyNewlyInitializedJobVertices
in interface ExecutionGraph
vertices
- The execution job vertices that are newly initialized.public void attachJobGraph(List<JobVertex> verticesToAttach, JobManagerJobMetricGroup jobManagerJobMetricGroup) throws JobException
attachJobGraph
in interface ExecutionGraph
JobException
public void initializeJobVertex(ExecutionJobVertex ejv, long createTimestamp, Map<IntermediateDataSetID,JobVertexInputInfo> jobVertexInputInfos, JobManagerJobMetricGroup jobManagerJobMetricGroup) throws JobException
ExecutionGraph
initializeJobVertex
in interface ExecutionGraph
ejv
- The execution job vertex that needs to be initialized.createTimestamp
- The timestamp for creating execution vertices, used to initialize the
first Execution with.jobVertexInputInfos
- The input infos of this job vertex.JobException
public void transitionToRunning()
transitionToRunning
in interface ExecutionGraph
public void cancel()
cancel
in interface ExecutionGraph
@VisibleForTesting protected FutureUtils.ConjunctFuture<Void> cancelVerticesAsync()
public void suspend(Throwable suspensionCause)
ExecutionGraph
The JobStatus will be directly set to JobStatus.SUSPENDED
iff the current state is
not a terminal state. All ExecutionJobVertices will be canceled and the onTerminalState() is
executed.
The JobStatus.SUSPENDED
state is a local terminal state which stops the execution
of the job but does not remove the job from the HA job store so that it can be recovered by
another JobManager.
suspend
in interface ExecutionGraph
suspensionCause
- Cause of the suspensionpublic void failGlobal(Throwable t)
InternalExecutionGraphAccessor
This global failure is meant to be triggered in cases where the consistency of the execution graph' state cannot be guaranteed any more (for example when catching unexpected exceptions that indicate a bug or an unexpected call race), and where a full restart is the safe way to get consistency back.
failGlobal
in interface InternalExecutionGraphAccessor
t
- The exception that caused the failure.public ArchivedExecutionConfig getArchivedExecutionConfig()
ArchivedExecutionConfig
.getArchivedExecutionConfig
in interface AccessExecutionGraph
public CompletableFuture<JobStatus> getTerminationFuture()
ExecutionGraph
ExecutionGraph
. The termination future is
completed with the terminal JobStatus
once the ExecutionGraph reaches this terminal
state and all Execution
have been terminated.getTerminationFuture
in interface ExecutionGraph
ExecutionGraph
.@VisibleForTesting public JobStatus waitUntilTerminal() throws InterruptedException
waitUntilTerminal
in interface ExecutionGraph
InterruptedException
public boolean transitionState(JobStatus current, JobStatus newState)
transitionState
in interface ExecutionGraph
public void incrementRestarts()
incrementRestarts
in interface ExecutionGraph
public void initFailureCause(Throwable t, long timestamp)
initFailureCause
in interface ExecutionGraph
public void jobVertexFinished()
jobVertexFinished
in interface InternalExecutionGraphAccessor
public void jobVertexUnFinished()
jobVertexUnFinished
in interface InternalExecutionGraphAccessor
public void failJob(Throwable cause, long timestamp)
failJob
in interface ExecutionGraph
public boolean updateState(TaskExecutionStateTransition state)
ExecutionGraph
updateState
in interface ExecutionGraph
state
- The state update.public Map<ExecutionAttemptID,Execution> getRegisteredExecutions()
getRegisteredExecutions
in interface ExecutionGraph
public void registerExecution(Execution exec)
registerExecution
in interface InternalExecutionGraphAccessor
public void deregisterExecution(Execution exec)
deregisterExecution
in interface InternalExecutionGraphAccessor
public void updateAccumulators(AccumulatorSnapshot accumulatorSnapshot)
ExecutionGraph
updateAccumulators
in interface ExecutionGraph
accumulatorSnapshot
- The serialized flink and user-defined accumulatorspublic void registerJobStatusListener(JobStatusListener listener)
registerJobStatusListener
in interface ExecutionGraph
public void notifyExecutionChange(Execution execution, ExecutionState previousState, ExecutionState newExecutionState)
notifyExecutionChange
in interface InternalExecutionGraphAccessor
public void notifySchedulerNgAboutInternalTaskFailure(ExecutionAttemptID attemptId, Throwable t, boolean cancelTask, boolean releasePartitions)
notifySchedulerNgAboutInternalTaskFailure
in interface InternalExecutionGraphAccessor
public void deleteBlobs(List<PermanentBlobKey> blobKeys)
deleteBlobs
in interface InternalExecutionGraphAccessor
public ShuffleMaster<?> getShuffleMaster()
getShuffleMaster
in interface InternalExecutionGraphAccessor
public JobMasterPartitionTracker getPartitionTracker()
getPartitionTracker
in interface InternalExecutionGraphAccessor
public ResultPartitionAvailabilityChecker getResultPartitionAvailabilityChecker()
getResultPartitionAvailabilityChecker
in interface ExecutionGraph
public PartitionGroupReleaseStrategy getPartitionGroupReleaseStrategy()
getPartitionGroupReleaseStrategy
in interface InternalExecutionGraphAccessor
public ExecutionDeploymentListener getExecutionDeploymentListener()
getExecutionDeploymentListener
in interface InternalExecutionGraphAccessor
public boolean isDynamic()
isDynamic
in interface InternalExecutionGraphAccessor
public Optional<String> findVertexWithAttempt(ExecutionAttemptID attemptId)
findVertexWithAttempt
in interface ExecutionGraph
public Optional<AccessExecution> findExecution(ExecutionAttemptID attemptId)
findExecution
in interface ExecutionGraph
public ExecutionGraphID getExecutionGraphID()
getExecutionGraphID
in interface InternalExecutionGraphAccessor
public List<ShuffleDescriptor> getClusterPartitionShuffleDescriptors(IntermediateDataSetID intermediateDataSetID)
InternalExecutionGraphAccessor
getClusterPartitionShuffleDescriptors
in interface InternalExecutionGraphAccessor
public MarkPartitionFinishedStrategy getMarkPartitionFinishedStrategy()
getMarkPartitionFinishedStrategy
in interface InternalExecutionGraphAccessor
public JobVertexInputInfo getJobVertexInputInfo(JobVertexID jobVertexId, IntermediateDataSetID resultId)
InternalExecutionGraphAccessor
getJobVertexInputInfo
in interface InternalExecutionGraphAccessor
jobVertexId
- the job vertex idresultId
- the input(intermediate result) idpublic TaskDeploymentDescriptorFactory getTaskDeploymentDescriptorFactory()
getTaskDeploymentDescriptorFactory
in interface InternalExecutionGraphAccessor
Copyright © 2014–2023 The Apache Software Foundation. All rights reserved.