public class AdaptiveBatchScheduler extends DefaultScheduler
executionDeployer, executionSlotAllocator, log
executionVertexVersioner, inputsLocationsRetriever, jobManagerJobMetricGroup, operatorCoordinatorHandler, stateLocationRetriever
Constructor and Description |
---|
AdaptiveBatchScheduler(org.slf4j.Logger log,
JobGraph jobGraph,
Executor ioExecutor,
Configuration jobMasterConfiguration,
java.util.function.Consumer<ComponentMainThreadExecutor> startUpAction,
ScheduledExecutor delayExecutor,
ClassLoader userCodeLoader,
CheckpointsCleaner checkpointsCleaner,
CheckpointRecoveryFactory checkpointRecoveryFactory,
JobManagerJobMetricGroup jobManagerJobMetricGroup,
SchedulingStrategyFactory schedulingStrategyFactory,
FailoverStrategy.Factory failoverStrategyFactory,
RestartBackoffTimeStrategy restartBackoffTimeStrategy,
ExecutionOperations executionOperations,
ExecutionVertexVersioner executionVertexVersioner,
ExecutionSlotAllocatorFactory executionSlotAllocatorFactory,
long initializationTimestamp,
ComponentMainThreadExecutor mainThreadExecutor,
JobStatusListener jobStatusListener,
Collection<FailureEnricher> failureEnrichers,
ExecutionGraphFactory executionGraphFactory,
ShuffleMaster<?> shuffleMaster,
Time rpcTimeout,
VertexParallelismAndInputInfosDecider vertexParallelismAndInputInfosDecider,
int defaultMaxParallelism,
JobManagerOptions.HybridPartitionDataConsumeConstraint hybridPartitionDataConsumeConstraint,
Map<JobVertexID,ForwardGroup> forwardGroupsByJobVertexId) |
Modifier and Type | Method and Description |
---|---|
void |
allocateSlotsAndDeploy(List<ExecutionVertexID> verticesToDeploy)
Allocate slots and deploy the vertex when slots are returned.
|
static VertexParallelismStore |
computeVertexParallelismStoreForDynamicGraph(Iterable<JobVertex> vertices,
int defaultMaxParallelism)
Compute the
VertexParallelismStore for all given vertices in a dynamic graph, which
will set defaults and ensure that the returned store contains valid parallelisms, with the
configured default max parallelism. |
protected MarkPartitionFinishedStrategy |
getMarkPartitionFinishedStrategy() |
void |
initializeVerticesIfPossible() |
protected void |
onTaskFinished(Execution execution,
IOMetrics ioMetrics) |
protected void |
resetForNewExecution(ExecutionVertexID executionVertexId) |
protected void |
startSchedulingInternal() |
cancelAllPendingSlotRequestsForVertex, cancelAllPendingSlotRequestsInternal, cancelExecution, createFailureHandlingResultSnapshot, getNumberOfRestarts, handleGlobalFailure, handleTaskFailure, onTaskFailed, recordTaskFailure
acknowledgeCheckpoint, archiveFromFailureHandlingResult, archiveGlobalFailure, cancel, closeAsync, computeVertexParallelismStore, computeVertexParallelismStore, computeVertexParallelismStore, computeVertexParallelismStore, declineCheckpoint, deliverCoordinationRequestToCoordinator, deliverOperatorEventToCoordinator, failJob, getDefaultMaxParallelism, getExceptionHistory, getExecutionGraph, getExecutionJobVertex, getExecutionVertex, getJobGraph, getJobTerminationFuture, getMainThreadExecutor, getResultPartitionAvailabilityChecker, getSchedulingTopology, notifyEndOfData, notifyKvStateRegistered, notifyKvStateUnregistered, registerJobMetrics, reportCheckpointMetrics, requestCheckpointStats, requestJob, requestJobDetails, requestJobStatus, requestKvStateLocation, requestNextInputSplit, requestPartitionState, resetForNewExecutions, restoreState, setGlobalFailureCause, startCheckpointScheduler, startScheduling, stopCheckpointScheduler, stopWithSavepoint, transitionExecutionGraphState, transitionToRunning, transitionToScheduled, triggerCheckpoint, triggerSavepoint, updateAccumulators, updateTaskExecutionState
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
requestJobResourceRequirements, updateJobResourceRequirements, updateTaskExecutionState
close
public AdaptiveBatchScheduler(org.slf4j.Logger log, JobGraph jobGraph, Executor ioExecutor, Configuration jobMasterConfiguration, java.util.function.Consumer<ComponentMainThreadExecutor> startUpAction, ScheduledExecutor delayExecutor, ClassLoader userCodeLoader, CheckpointsCleaner checkpointsCleaner, CheckpointRecoveryFactory checkpointRecoveryFactory, JobManagerJobMetricGroup jobManagerJobMetricGroup, SchedulingStrategyFactory schedulingStrategyFactory, FailoverStrategy.Factory failoverStrategyFactory, RestartBackoffTimeStrategy restartBackoffTimeStrategy, ExecutionOperations executionOperations, ExecutionVertexVersioner executionVertexVersioner, ExecutionSlotAllocatorFactory executionSlotAllocatorFactory, long initializationTimestamp, ComponentMainThreadExecutor mainThreadExecutor, JobStatusListener jobStatusListener, Collection<FailureEnricher> failureEnrichers, ExecutionGraphFactory executionGraphFactory, ShuffleMaster<?> shuffleMaster, Time rpcTimeout, VertexParallelismAndInputInfosDecider vertexParallelismAndInputInfosDecider, int defaultMaxParallelism, JobManagerOptions.HybridPartitionDataConsumeConstraint hybridPartitionDataConsumeConstraint, Map<JobVertexID,ForwardGroup> forwardGroupsByJobVertexId) throws Exception
Exception
protected void startSchedulingInternal()
startSchedulingInternal
in class DefaultScheduler
protected void onTaskFinished(Execution execution, IOMetrics ioMetrics)
onTaskFinished
in class DefaultScheduler
public void allocateSlotsAndDeploy(List<ExecutionVertexID> verticesToDeploy)
SchedulerOperations
allocateSlotsAndDeploy
in interface SchedulerOperations
allocateSlotsAndDeploy
in class DefaultScheduler
verticesToDeploy
- The execution vertices to deployprotected void resetForNewExecution(ExecutionVertexID executionVertexId)
resetForNewExecution
in class SchedulerBase
protected MarkPartitionFinishedStrategy getMarkPartitionFinishedStrategy()
getMarkPartitionFinishedStrategy
in class SchedulerBase
@VisibleForTesting public void initializeVerticesIfPossible()
@VisibleForTesting public static VertexParallelismStore computeVertexParallelismStoreForDynamicGraph(Iterable<JobVertex> vertices, int defaultMaxParallelism)
VertexParallelismStore
for all given vertices in a dynamic graph, which
will set defaults and ensure that the returned store contains valid parallelisms, with the
configured default max parallelism.vertices
- the vertices to compute parallelism fordefaultMaxParallelism
- the global default max parallelismCopyright © 2014–2023 The Apache Software Foundation. All rights reserved.