Package org.apache.flink.streaming.api.graph

Class StreamingJobGraphGenerator

java.lang.Object

org.apache.flink.streaming.api.graph.StreamingJobGraphGenerator

@Internal
public class StreamingJobGraphGenerator
extends Object

The StreamingJobGraphGenerator converts a StreamGraph into a JobGraph.

Method Summary

Modifier and Type	Method	Description
static void	addVertexIndexPrefixInVertexName(JobVertexBuildContext jobVertexBuildContext, AtomicInteger vertexIndexId, JobGraph jobGraph)
static void	configureCheckpointing(StreamGraph streamGraph, JobGraph jobGraph)
static void	connect(Integer headOfChain, StreamEdge edge, NonChainedOutput output, Map<Integer,JobVertex> jobVertices, JobVertexBuildContext jobVertexBuildContext)
static ResourceSpec	createChainedMinResources(Integer vertexID, List<StreamEdge> chainedOutputs, OperatorChainInfo operatorChainInfo, JobVertexBuildContext jobVertexBuildContext)
static String	createChainedName(Integer vertexID, List<StreamEdge> chainedOutputs, Optional<OperatorChainInfo> operatorChainInfo, Map<Integer,String> chainedNames, JobVertexBuildContext jobVertexBuildContext)
static ResourceSpec	createChainedPreferredResources(Integer vertexID, List<StreamEdge> chainedOutputs, OperatorChainInfo operatorChainInfo, JobVertexBuildContext jobVertexBuildContext)
static JobGraph	createJobGraph(ClassLoader userClassLoader, StreamGraph streamGraph, JobID jobID)
static JobGraph	createJobGraph(StreamGraph streamGraph)
static ResultPartitionType	determineUndefinedResultPartitionType(StreamEdge edge, JobVertexBuildContext jobVertexBuildContext)
static OperatorID	generateOperatorID(String operatorUid)	Creates an instance of OperatorID based on the provided operator unique identifier (UID).
static boolean	isChainable(StreamEdge edge, StreamGraph streamGraph)
static boolean	isChainableInput(StreamEdge edge, StreamGraph streamGraph)
static void	markSupportingConcurrentExecutionAttempts(JobVertexBuildContext jobVertexBuildContext)
static void	preValidate(StreamGraph streamGraph, ClassLoader userClassloader)
static void	serializeOperatorCoordinatorsAndStreamConfig(JobGraph jobGraph, Executor serializationExecutor, JobVertexBuildContext jobVertexBuildContext)
static void	setAllOperatorNonChainedOutputsConfigs(Map<Integer,Map<StreamEdge,NonChainedOutput>> opIntermediateOutputs, JobVertexBuildContext jobVertexBuildContext)
static void	setManagedMemoryFraction(JobVertexBuildContext jobVertexBuildContext)
static void	setOperatorChainedOutputsConfig(StreamConfig config, List<StreamEdge> chainableOutputs, JobVertexBuildContext jobVertexBuildContext)
static void	setOperatorConfig(Integer vertexId, StreamConfig config, Map<Integer,ChainedSourceInfo> chainedSources, JobVertexBuildContext jobVertexBuildContext)
static void	setPhysicalEdges(JobVertexBuildContext jobVertexBuildContext)
static void	setSlotSharingAndCoLocation(JobGraph jobGraph, JobVertexBuildContext jobVertexBuildContext)
static void	setVertexDescription(JobVertexBuildContext jobVertexBuildContext)
static void	tryConvertPartitionerForDynamicGraph(List<StreamEdge> chainableOutputs, List<StreamEdge> nonChainableOutputs, JobVertexBuildContext jobVertexBuildContext)
static void	validateHybridShuffleExecuteInBatchMode(JobVertexBuildContext jobVertexBuildContext)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Method Detail

createJobGraph

@VisibleForTesting
public static JobGraph createJobGraph(StreamGraph streamGraph)

createJobGraph

public static JobGraph createJobGraph(ClassLoader userClassLoader,
                                      StreamGraph streamGraph,
                                      @Nullable
                                      JobID jobID)

serializeOperatorCoordinatorsAndStreamConfig

public static void serializeOperatorCoordinatorsAndStreamConfig(JobGraph jobGraph,
                                                                Executor serializationExecutor,
                                                                JobVertexBuildContext jobVertexBuildContext)

generateOperatorID

public static OperatorID generateOperatorID(String operatorUid)

Creates an instance of OperatorID based on the provided operator unique identifier (UID).

Parameters:

operatorUid - the unique identifier of the operator, used to generate the hash

Returns:

a new OperatorID instance generated from the specified operator UID

addVertexIndexPrefixInVertexName

public static void addVertexIndexPrefixInVertexName(JobVertexBuildContext jobVertexBuildContext,
                                                    AtomicInteger vertexIndexId,
                                                    JobGraph jobGraph)

setVertexDescription

public static void setVertexDescription(JobVertexBuildContext jobVertexBuildContext)

preValidate

public static void preValidate(StreamGraph streamGraph,
                               ClassLoader userClassloader)

setPhysicalEdges

public static void setPhysicalEdges(JobVertexBuildContext jobVertexBuildContext)

createChainedName

public static String createChainedName(Integer vertexID,
                                       List<StreamEdge> chainedOutputs,
                                       Optional<OperatorChainInfo> operatorChainInfo,
                                       Map<Integer,String> chainedNames,
                                       JobVertexBuildContext jobVertexBuildContext)

createChainedMinResources

public static ResourceSpec createChainedMinResources(Integer vertexID,
                                                     List<StreamEdge> chainedOutputs,
                                                     OperatorChainInfo operatorChainInfo,
                                                     JobVertexBuildContext jobVertexBuildContext)

createChainedPreferredResources

public static ResourceSpec createChainedPreferredResources(Integer vertexID,
                                                           List<StreamEdge> chainedOutputs,
                                                           OperatorChainInfo operatorChainInfo,
                                                           JobVertexBuildContext jobVertexBuildContext)

setOperatorConfig

public static void setOperatorConfig(Integer vertexId,
                                     StreamConfig config,
                                     Map<Integer,ChainedSourceInfo> chainedSources,
                                     JobVertexBuildContext jobVertexBuildContext)

setOperatorChainedOutputsConfig

public static void setOperatorChainedOutputsConfig(StreamConfig config,
                                                   List<StreamEdge> chainableOutputs,
                                                   JobVertexBuildContext jobVertexBuildContext)

setAllOperatorNonChainedOutputsConfigs

public static void setAllOperatorNonChainedOutputsConfigs(Map<Integer,Map<StreamEdge,NonChainedOutput>> opIntermediateOutputs,
                                                          JobVertexBuildContext jobVertexBuildContext)

tryConvertPartitionerForDynamicGraph

public static void tryConvertPartitionerForDynamicGraph(List<StreamEdge> chainableOutputs,
                                                        List<StreamEdge> nonChainableOutputs,
                                                        JobVertexBuildContext jobVertexBuildContext)

connect

public static void connect(Integer headOfChain,
                           StreamEdge edge,
                           NonChainedOutput output,
                           Map<Integer,JobVertex> jobVertices,
                           JobVertexBuildContext jobVertexBuildContext)

determineUndefinedResultPartitionType

public static ResultPartitionType determineUndefinedResultPartitionType(StreamEdge edge,
                                                                        JobVertexBuildContext jobVertexBuildContext)

isChainable

public static boolean isChainable(StreamEdge edge,
                                  StreamGraph streamGraph)

isChainableInput

public static boolean isChainableInput(StreamEdge edge,
                                       StreamGraph streamGraph)

markSupportingConcurrentExecutionAttempts

public static void markSupportingConcurrentExecutionAttempts(JobVertexBuildContext jobVertexBuildContext)

setSlotSharingAndCoLocation

public static void setSlotSharingAndCoLocation(JobGraph jobGraph,
                                               JobVertexBuildContext jobVertexBuildContext)

validateHybridShuffleExecuteInBatchMode

public static void validateHybridShuffleExecuteInBatchMode(JobVertexBuildContext jobVertexBuildContext)

setManagedMemoryFraction

public static void setManagedMemoryFraction(JobVertexBuildContext jobVertexBuildContext)

configureCheckpointing

public static void configureCheckpointing(StreamGraph streamGraph,
                                          JobGraph jobGraph)