public class YarnResourceManager extends ActiveResourceManager<YarnWorkerNode> implements org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler, org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler
ActiveResourceManager.PendingWorkerNums
RpcEndpoint.MainThreadExecutor
env, flinkClientConfig, flinkConfig
RESOURCE_MANAGER_NAME
log, rpcServer
Constructor and Description |
---|
YarnResourceManager(RpcService rpcService,
ResourceID resourceId,
Configuration flinkConfig,
Map<String,String> env,
HighAvailabilityServices highAvailabilityServices,
HeartbeatServices heartbeatServices,
SlotManager slotManager,
ResourceManagerPartitionTrackerFactory clusterPartitionTrackerFactory,
JobLeaderIdService jobLeaderIdService,
ClusterInformation clusterInformation,
FatalErrorHandler fatalErrorHandler,
String webInterfaceUrl,
ResourceManagerMetricGroup resourceManagerMetricGroup) |
Modifier and Type | Method and Description |
---|---|
protected org.apache.hadoop.yarn.client.api.async.NMClientAsync |
createAndStartNodeManagerClient(org.apache.hadoop.yarn.conf.YarnConfiguration yarnConfiguration) |
protected org.apache.hadoop.yarn.client.api.async.AMRMClientAsync<org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest> |
createAndStartResourceManagerClient(org.apache.hadoop.yarn.conf.YarnConfiguration yarnConfiguration,
int yarnHeartbeatIntervalMillis,
String webInterfaceUrl) |
float |
getProgress() |
protected void |
initialize()
Initializes the framework specific components.
|
protected void |
internalDeregisterApplication(ApplicationStatus finalStatus,
String diagnostics)
The framework specific code to deregister the application.
|
protected Configuration |
loadClientConfiguration() |
void |
onContainersAllocated(List<org.apache.hadoop.yarn.api.records.Container> containers) |
void |
onContainersCompleted(List<org.apache.hadoop.yarn.api.records.ContainerStatus> statuses) |
void |
onContainerStarted(org.apache.hadoop.yarn.api.records.ContainerId containerId,
Map<String,ByteBuffer> map) |
void |
onContainerStatusReceived(org.apache.hadoop.yarn.api.records.ContainerId containerId,
org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus) |
void |
onContainerStopped(org.apache.hadoop.yarn.api.records.ContainerId containerId) |
void |
onError(Throwable error) |
void |
onGetContainerStatusError(org.apache.hadoop.yarn.api.records.ContainerId containerId,
Throwable throwable) |
void |
onNodesUpdated(List<org.apache.hadoop.yarn.api.records.NodeReport> list) |
void |
onShutdownRequest() |
void |
onStartContainerError(org.apache.hadoop.yarn.api.records.ContainerId containerId,
Throwable t) |
CompletableFuture<Void> |
onStop()
User overridable callback which is called from
RpcEndpoint.internalCallOnStop() . |
void |
onStopContainerError(org.apache.hadoop.yarn.api.records.ContainerId containerId,
Throwable throwable) |
boolean |
startNewWorker(WorkerResourceSpec workerResourceSpec)
Allocates a resource using the worker resource specification.
|
boolean |
stopWorker(YarnWorkerNode workerNode)
Stops the given worker.
|
protected YarnWorkerNode |
workerStarted(ResourceID resourceID)
Callback when a worker was started.
|
getNumRequestedNotAllocatedWorkers, getNumRequestedNotAllocatedWorkersFor, getNumRequestedNotRegisteredWorkers, getNumRequestedNotRegisteredWorkersFor, getStopTerminationFutureOrCompletedExceptionally, notifyAllocatedWorkerStopped, notifyNewWorkerAllocated, notifyNewWorkerAllocationFailed, notifyNewWorkerRequested, onTaskManagerRegistration
cancelSlotRequest, clearStateAsync, closeJobManagerConnection, closeTaskManagerConnection, deregisterApplication, disconnectJobManager, disconnectTaskManager, getNumberOfRegisteredTaskManagers, getNumberRequiredTaskManagers, getRequiredResources, grantLeadership, handleError, heartbeatFromJobManager, heartbeatFromTaskManager, jobLeaderLostLeadership, listDataSets, notifySlotAvailable, onFatalError, onStart, prepareLeadershipAsync, registerJobManager, registerTaskExecutor, releaseClusterPartitions, releaseResource, removeJob, requestResourceOverview, requestSlot, requestTaskManagerFileUploadByName, requestTaskManagerFileUploadByType, requestTaskManagerInfo, requestTaskManagerInfo, requestTaskManagerLogList, requestTaskManagerMetricQueryServiceAddresses, requestThreadDump, revokeLeadership, sendSlotReport, setFailUnfulfillableRequest, startServicesOnLeadership
callAsyncWithoutFencing, getFencingToken, getMainThreadExecutor, getUnfencedMainThreadExecutor, runAsyncWithoutFencing, setFencingToken
callAsync, closeAsync, getAddress, getEndpointId, getHostname, getRpcService, getSelfGateway, getTerminationFuture, internalCallOnStart, internalCallOnStop, isRunning, runAsync, scheduleRunAsync, scheduleRunAsync, start, stop, validateRunsInMainThread
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
getFencingToken
getAddress, getHostname
getDescription
close
public YarnResourceManager(RpcService rpcService, ResourceID resourceId, Configuration flinkConfig, Map<String,String> env, HighAvailabilityServices highAvailabilityServices, HeartbeatServices heartbeatServices, SlotManager slotManager, ResourceManagerPartitionTrackerFactory clusterPartitionTrackerFactory, JobLeaderIdService jobLeaderIdService, ClusterInformation clusterInformation, FatalErrorHandler fatalErrorHandler, @Nullable String webInterfaceUrl, ResourceManagerMetricGroup resourceManagerMetricGroup)
protected org.apache.hadoop.yarn.client.api.async.AMRMClientAsync<org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest> createAndStartResourceManagerClient(org.apache.hadoop.yarn.conf.YarnConfiguration yarnConfiguration, int yarnHeartbeatIntervalMillis, @Nullable String webInterfaceUrl) throws Exception
Exception
protected org.apache.hadoop.yarn.client.api.async.NMClientAsync createAndStartNodeManagerClient(org.apache.hadoop.yarn.conf.YarnConfiguration yarnConfiguration)
protected Configuration loadClientConfiguration()
loadClientConfiguration
in class ActiveResourceManager<YarnWorkerNode>
protected void initialize() throws ResourceManagerException
ResourceManager
initialize
in class ResourceManager<YarnWorkerNode>
ResourceManagerException
- which occurs during initialization and causes the resource
manager to fail.public CompletableFuture<Void> onStop()
RpcEndpoint
RpcEndpoint.internalCallOnStop()
.
This method is called when the RpcEndpoint is being shut down. The method is guaranteed to be executed in the main thread context and can be used to clean up internal state.
IMPORTANT: This method should never be called directly by the user.
onStop
in class ResourceManager<YarnWorkerNode>
protected void internalDeregisterApplication(ApplicationStatus finalStatus, @Nullable String diagnostics)
ResourceManager
This method also needs to make sure all pending containers that are not registered yet are returned.
internalDeregisterApplication
in class ResourceManager<YarnWorkerNode>
finalStatus
- The application status to report.diagnostics
- A diagnostics message or null
.public boolean startNewWorker(WorkerResourceSpec workerResourceSpec)
ResourceManager
startNewWorker
in class ResourceManager<YarnWorkerNode>
workerResourceSpec
- workerResourceSpec specifies the size of the to be allocated
resourcepublic boolean stopWorker(YarnWorkerNode workerNode)
ResourceManager
stopWorker
in class ResourceManager<YarnWorkerNode>
workerNode
- The worker.protected YarnWorkerNode workerStarted(ResourceID resourceID)
ResourceManager
workerStarted
in class ResourceManager<YarnWorkerNode>
resourceID
- The worker resource idpublic float getProgress()
getProgress
in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler
public void onContainersCompleted(List<org.apache.hadoop.yarn.api.records.ContainerStatus> statuses)
onContainersCompleted
in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler
public void onContainersAllocated(List<org.apache.hadoop.yarn.api.records.Container> containers)
onContainersAllocated
in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler
public void onShutdownRequest()
onShutdownRequest
in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler
public void onNodesUpdated(List<org.apache.hadoop.yarn.api.records.NodeReport> list)
onNodesUpdated
in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler
public void onError(Throwable error)
onError
in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler
public void onContainerStarted(org.apache.hadoop.yarn.api.records.ContainerId containerId, Map<String,ByteBuffer> map)
onContainerStarted
in interface org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler
public void onContainerStatusReceived(org.apache.hadoop.yarn.api.records.ContainerId containerId, org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus)
onContainerStatusReceived
in interface org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler
public void onContainerStopped(org.apache.hadoop.yarn.api.records.ContainerId containerId)
onContainerStopped
in interface org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler
public void onStartContainerError(org.apache.hadoop.yarn.api.records.ContainerId containerId, Throwable t)
onStartContainerError
in interface org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler
public void onGetContainerStatusError(org.apache.hadoop.yarn.api.records.ContainerId containerId, Throwable throwable)
onGetContainerStatusError
in interface org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler
public void onStopContainerError(org.apache.hadoop.yarn.api.records.ContainerId containerId, Throwable throwable)
onStopContainerError
in interface org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler
Copyright © 2014–2021 The Apache Software Foundation. All rights reserved.