public abstract class AbstractYarnClusterDescriptor extends Object implements ClusterDescriptor<YarnClusterClient>
YarnClusterClient
.Modifier and Type | Field and Description |
---|---|
protected List<File> |
shipFiles
Lazily initialized list of files to ship
|
Constructor and Description |
---|
AbstractYarnClusterDescriptor() |
Modifier and Type | Method and Description |
---|---|
protected void |
addLibFolderToShipFiles(Collection<File> effectiveShipFiles) |
void |
addShipFiles(List<File> shipFiles) |
protected YarnClusterClient |
createYarnClusterClient(AbstractYarnClusterDescriptor descriptor,
org.apache.hadoop.yarn.client.api.YarnClient yarnClient,
org.apache.hadoop.yarn.api.records.ApplicationReport report,
Configuration flinkConfiguration,
boolean perJobCluster)
Creates a YarnClusterClient; may be overriden in tests
|
YarnClusterClient |
deploy()
Triggers deployment of a cluster
|
protected YarnClusterClient |
deployInternal()
This method will block until the ApplicationMaster/JobManager have been
deployed on YARN.
|
protected abstract Class<?> |
getApplicationMasterClass()
The class to bootstrap the application master of the Yarn cluster (runs main method).
|
String |
getClusterDescription()
Returns a String containing details about the cluster (NodeManagers, available memory, ...)
|
String |
getDynamicPropertiesEncoded() |
Configuration |
getFlinkConfiguration() |
int |
getTaskManagerCount() |
int |
getTaskManagerSlots() |
protected org.apache.hadoop.yarn.client.api.YarnClient |
getYarnClient()
Gets a Hadoop Yarn client
|
String |
getZookeeperNamespace() |
boolean |
hasUserJarFiles(List<URL> requiredJarFiles)
Returns true if the descriptor has the job jars to include in the classpath.
|
boolean |
isDetachedMode() |
YarnClusterClient |
retrieve(String applicationID)
Retrieves an existing Flink Cluster.
|
void |
setConfigurationDirectory(String configurationDirectory) |
void |
setConfigurationFilePath(org.apache.hadoop.fs.Path confPath) |
void |
setDetachedMode(boolean detachedMode) |
void |
setDynamicPropertiesEncoded(String dynamicPropertiesEncoded) |
void |
setFlinkConfiguration(Configuration conf) |
void |
setJobManagerMemory(int memoryMb) |
void |
setLocalJarPath(org.apache.hadoop.fs.Path localJarPath) |
void |
setName(String name) |
void |
setProvidedUserJarFiles(List<URL> userJarFiles)
Sets the user jar which is included in the system classloader of all nodes.
|
void |
setQueue(String queue) |
void |
setTaskManagerCount(int tmCount) |
void |
setTaskManagerMemory(int memoryMb) |
void |
setTaskManagerSlots(int slots) |
protected org.apache.hadoop.yarn.api.records.ContainerLaunchContext |
setupApplicationMasterContainer(boolean hasLogback,
boolean hasLog4j,
boolean hasKrb5) |
void |
setZookeeperNamespace(String zookeeperNamespace) |
org.apache.hadoop.yarn.api.records.ApplicationReport |
startAppMaster(JobGraph jobGraph,
org.apache.hadoop.yarn.client.api.YarnClient yarnClient,
org.apache.hadoop.yarn.client.api.YarnClientApplication yarnApplication) |
protected abstract Class<?> getApplicationMasterClass()
public void setJobManagerMemory(int memoryMb)
public void setTaskManagerMemory(int memoryMb)
public void setFlinkConfiguration(Configuration conf)
public Configuration getFlinkConfiguration()
public void setTaskManagerSlots(int slots)
public int getTaskManagerSlots()
public void setQueue(String queue)
public void setLocalJarPath(org.apache.hadoop.fs.Path localJarPath)
public void setConfigurationFilePath(org.apache.hadoop.fs.Path confPath)
public void setConfigurationDirectory(String configurationDirectory)
public void setTaskManagerCount(int tmCount)
public int getTaskManagerCount()
public void setDynamicPropertiesEncoded(String dynamicPropertiesEncoded)
public boolean hasUserJarFiles(List<URL> requiredJarFiles)
public void setProvidedUserJarFiles(List<URL> userJarFiles)
public String getDynamicPropertiesEncoded()
public void setDetachedMode(boolean detachedMode)
public boolean isDetachedMode()
public String getZookeeperNamespace()
public void setZookeeperNamespace(String zookeeperNamespace)
protected org.apache.hadoop.yarn.client.api.YarnClient getYarnClient()
public YarnClusterClient retrieve(String applicationID)
ClusterDescriptor
retrieve
in interface ClusterDescriptor<YarnClusterClient>
applicationID
- The unique application identifier of the running clusterpublic YarnClusterClient deploy()
ClusterDescriptor
deploy
in interface ClusterDescriptor<YarnClusterClient>
protected YarnClusterClient deployInternal() throws Exception
Exception
public org.apache.hadoop.yarn.api.records.ApplicationReport startAppMaster(JobGraph jobGraph, org.apache.hadoop.yarn.client.api.YarnClient yarnClient, org.apache.hadoop.yarn.client.api.YarnClientApplication yarnApplication) throws Exception
Exception
public String getClusterDescription()
ClusterDescriptor
getClusterDescription
in interface ClusterDescriptor<YarnClusterClient>
public void setName(String name)
protected void addLibFolderToShipFiles(Collection<File> effectiveShipFiles)
protected org.apache.hadoop.yarn.api.records.ContainerLaunchContext setupApplicationMasterContainer(boolean hasLogback, boolean hasLog4j, boolean hasKrb5)
protected YarnClusterClient createYarnClusterClient(AbstractYarnClusterDescriptor descriptor, org.apache.hadoop.yarn.client.api.YarnClient yarnClient, org.apache.hadoop.yarn.api.records.ApplicationReport report, Configuration flinkConfiguration, boolean perJobCluster) throws Exception
Exception
Copyright © 2014–2018 The Apache Software Foundation. All rights reserved.