org.apache.flink.yarn

Class YarnResourceManager

java.lang.Object

org.apache.flink.runtime.rpc.RpcEndpoint

org.apache.flink.runtime.rpc.FencedRpcEndpoint<ResourceManagerId>

org.apache.flink.runtime.resourcemanager.ResourceManager<YarnWorkerNode>

org.apache.flink.yarn.YarnResourceManager

All Implemented Interfaces:

AutoCloseable, LeaderContender, ResourceManagerGateway, FencedRpcGateway<ResourceManagerId>, RpcGateway, AutoCloseableAsync, org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler, org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler

public class YarnResourceManager
extends ResourceManager<YarnWorkerNode>
implements org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler, org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler

The yarn implementation of the resource manager. Used when the system is started via the resource framework YARN.

Nested Class Summary

Nested classes/interfaces inherited from class org.apache.flink.runtime.rpc.RpcEndpoint

RpcEndpoint.MainThreadExecutor

Field Summary

Fields inherited from class org.apache.flink.runtime.resourcemanager.ResourceManager

RESOURCE_MANAGER_NAME

Fields inherited from class org.apache.flink.runtime.rpc.RpcEndpoint

log, rpcServer

Constructor Summary

Constructors

Constructor and Description

YarnResourceManager(RpcService rpcService, String resourceManagerEndpointId, ResourceID resourceId, Configuration flinkConfig, Map<String,String> env, HighAvailabilityServices highAvailabilityServices, HeartbeatServices heartbeatServices, SlotManager slotManager, MetricRegistry metricRegistry, JobLeaderIdService jobLeaderIdService, ClusterInformation clusterInformation, FatalErrorHandler fatalErrorHandler, String webInterfaceUrl, JobManagerMetricGroup jobManagerMetricGroup)

Method Summary

Modifier and Type	Method and Description
protected org.apache.hadoop.yarn.client.api.async.NMClientAsync	createAndStartNodeManagerClient(org.apache.hadoop.yarn.conf.YarnConfiguration yarnConfiguration)
protected org.apache.hadoop.yarn.client.api.async.AMRMClientAsync<org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest>	createAndStartResourceManagerClient(org.apache.hadoop.yarn.conf.YarnConfiguration yarnConfiguration, int yarnHeartbeatIntervalMillis, String webInterfaceUrl)
float	getProgress()
protected void	initialize() Initializes the framework specific components.
protected void	internalDeregisterApplication(ApplicationStatus finalStatus, String diagnostics) The framework specific code to deregister the application.
void	onContainersAllocated(List<org.apache.hadoop.yarn.api.records.Container> containers)
void	onContainersCompleted(List<org.apache.hadoop.yarn.api.records.ContainerStatus> statuses)
void	onContainerStarted(org.apache.hadoop.yarn.api.records.ContainerId containerId, Map<String,ByteBuffer> map)
void	onContainerStatusReceived(org.apache.hadoop.yarn.api.records.ContainerId containerId, org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus)
void	onContainerStopped(org.apache.hadoop.yarn.api.records.ContainerId containerId)
void	onError(Throwable error)
void	onGetContainerStatusError(org.apache.hadoop.yarn.api.records.ContainerId containerId, Throwable throwable)
void	onNodesUpdated(List<org.apache.hadoop.yarn.api.records.NodeReport> list)
void	onShutdownRequest()
void	onStartContainerError(org.apache.hadoop.yarn.api.records.ContainerId containerId, Throwable t)
CompletableFuture<Void>	onStop() User overridable callback.
void	onStopContainerError(org.apache.hadoop.yarn.api.records.ContainerId containerId, Throwable throwable)
Collection<ResourceProfile>	startNewWorker(ResourceProfile resourceProfile) Allocates a resource using the resource profile.
boolean	stopWorker(YarnWorkerNode workerNode) Stops the given worker.
protected YarnWorkerNode	workerStarted(ResourceID resourceID) Callback when a worker was started.

Methods inherited from class org.apache.flink.runtime.resourcemanager.ResourceManager

cancelSlotRequest, clearStateAsync, closeJobManagerConnection, closeTaskManagerConnection, createSlotsPerWorker, deregisterApplication, disconnectJobManager, disconnectTaskManager, getNumberOfRegisteredTaskManagers, getNumberRequiredTaskManagerSlots, grantLeadership, handleError, heartbeatFromJobManager, heartbeatFromTaskManager, jobLeaderLostLeadership, notifySlotAvailable, onFatalError, onStart, prepareLeadershipAsync, registerInfoMessageListener, registerJobManager, registerTaskExecutor, releaseResource, removeJob, requestResourceOverview, requestSlot, requestTaskManagerFileUpload, requestTaskManagerInfo, requestTaskManagerInfo, requestTaskManagerMetricQueryServicePaths, revokeLeadership, sendInfoMessage, sendSlotReport, unRegisterInfoMessageListener

Methods inherited from class org.apache.flink.runtime.rpc.FencedRpcEndpoint

callAsyncWithoutFencing, getFencingToken, getMainThreadExecutor, getUnfencedMainThreadExecutor, runAsyncWithoutFencing, setFencingToken

Methods inherited from class org.apache.flink.runtime.rpc.RpcEndpoint

callAsync, closeAsync, getAddress, getEndpointId, getHostname, getRpcService, getSelfGateway, getTerminationFuture, runAsync, scheduleRunAsync, scheduleRunAsync, start, stop, validateRunsInMainThread

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.flink.runtime.rpc.FencedRpcGateway

getFencingToken

Methods inherited from interface org.apache.flink.runtime.rpc.RpcGateway

getAddress, getHostname

Methods inherited from interface org.apache.flink.runtime.leaderelection.LeaderContender

getAddress

Methods inherited from interface org.apache.flink.util.AutoCloseableAsync

close

Constructor Detail

YarnResourceManager

public YarnResourceManager(RpcService rpcService,
                           String resourceManagerEndpointId,
                           ResourceID resourceId,
                           Configuration flinkConfig,
                           Map<String,String> env,
                           HighAvailabilityServices highAvailabilityServices,
                           HeartbeatServices heartbeatServices,
                           SlotManager slotManager,
                           MetricRegistry metricRegistry,
                           JobLeaderIdService jobLeaderIdService,
                           ClusterInformation clusterInformation,
                           FatalErrorHandler fatalErrorHandler,
                           @Nullable
                           String webInterfaceUrl,
                           JobManagerMetricGroup jobManagerMetricGroup)

Method Detail

createAndStartResourceManagerClient

protected org.apache.hadoop.yarn.client.api.async.AMRMClientAsync<org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest> createAndStartResourceManagerClient(org.apache.hadoop.yarn.conf.YarnConfiguration yarnConfiguration,
                                                                                                                                                                     int yarnHeartbeatIntervalMillis,
                                                                                                                                                                     @Nullable
                                                                                                                                                                     String webInterfaceUrl)
                                                                                                                                                              throws Exception

Throws:

Exception

createAndStartNodeManagerClient

protected org.apache.hadoop.yarn.client.api.async.NMClientAsync createAndStartNodeManagerClient(org.apache.hadoop.yarn.conf.YarnConfiguration yarnConfiguration)

initialize

protected void initialize()
                   throws ResourceManagerException

Description copied from class: ResourceManager

Initializes the framework specific components.

Specified by:

initialize in class ResourceManager<YarnWorkerNode>

Throws:

ResourceManagerException - which occurs during initialization and causes the resource manager to fail.

onStop

public CompletableFuture<Void> onStop()

Description copied from class: RpcEndpoint

User overridable callback.

This method is called when the RpcEndpoint is being shut down. The method is guaranteed to be executed in the main thread context and can be used to clean up internal state.

IMPORTANT: This method should never be called directly by the user.

Overrides:

onStop in class ResourceManager<YarnWorkerNode>

Returns:

Future which is completed once all post stop actions are completed. If an error occurs this future is completed exceptionally

internalDeregisterApplication

protected void internalDeregisterApplication(ApplicationStatus finalStatus,
                                             @Nullable
                                             String diagnostics)

Description copied from class: ResourceManager

The framework specific code to deregister the application. This should report the application's final status and shut down the resource manager cleanly.

This method also needs to make sure all pending containers that are not registered yet are returned.

Specified by:

internalDeregisterApplication in class ResourceManager<YarnWorkerNode>

Parameters:

finalStatus - The application status to report.

diagnostics - A diagnostics message or null.

startNewWorker

public Collection<ResourceProfile> startNewWorker(ResourceProfile resourceProfile)

Description copied from class: ResourceManager

Allocates a resource using the resource profile.

Specified by:

startNewWorker in class ResourceManager<YarnWorkerNode>

Parameters:

resourceProfile - The resource description

Returns:

Collection of ResourceProfile describing the launched slots

stopWorker

public boolean stopWorker(YarnWorkerNode workerNode)

Description copied from class: ResourceManager

Stops the given worker.

Specified by:

stopWorker in class ResourceManager<YarnWorkerNode>

Parameters:

workerNode - The worker.

Returns:

True if the worker was stopped, otherwise false

workerStarted

protected YarnWorkerNode workerStarted(ResourceID resourceID)

Description copied from class: ResourceManager

Callback when a worker was started.

Specified by:

workerStarted in class ResourceManager<YarnWorkerNode>

Parameters:

resourceID - The worker resource id

getProgress

public float getProgress()

Specified by:

getProgress in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onContainersCompleted

public void onContainersCompleted(List<org.apache.hadoop.yarn.api.records.ContainerStatus> statuses)

Specified by:

onContainersCompleted in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onContainersAllocated

public void onContainersAllocated(List<org.apache.hadoop.yarn.api.records.Container> containers)

Specified by:

onContainersAllocated in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onShutdownRequest

public void onShutdownRequest()

Specified by:

onShutdownRequest in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onNodesUpdated

public void onNodesUpdated(List<org.apache.hadoop.yarn.api.records.NodeReport> list)

Specified by:

onNodesUpdated in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onError

public void onError(Throwable error)

Specified by:

onError in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onContainerStarted

public void onContainerStarted(org.apache.hadoop.yarn.api.records.ContainerId containerId,
                               Map<String,ByteBuffer> map)

Specified by:

onContainerStarted in interface org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler

onContainerStatusReceived

public void onContainerStatusReceived(org.apache.hadoop.yarn.api.records.ContainerId containerId,
                                      org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus)

Specified by:

onContainerStatusReceived in interface org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler

onContainerStopped

public void onContainerStopped(org.apache.hadoop.yarn.api.records.ContainerId containerId)

Specified by:

onContainerStopped in interface org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler

onStartContainerError

public void onStartContainerError(org.apache.hadoop.yarn.api.records.ContainerId containerId,
                                  Throwable t)

Specified by:

onStartContainerError in interface org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler

onGetContainerStatusError

public void onGetContainerStatusError(org.apache.hadoop.yarn.api.records.ContainerId containerId,
                                      Throwable throwable)

Specified by:

onGetContainerStatusError in interface org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler

onStopContainerError

public void onStopContainerError(org.apache.hadoop.yarn.api.records.ContainerId containerId,
                                 Throwable throwable)

Specified by:

onStopContainerError in interface org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler