org.apache.flink.streaming.api.graph

Class StreamGraph

java.lang.Object

org.apache.flink.optimizer.plan.StreamingPlan

org.apache.flink.streaming.api.graph.StreamGraph

All Implemented Interfaces:

FlinkPlan

@Internal
public class StreamGraph
extends StreamingPlan

Class representing the streaming topology. It contains all the information necessary to build the jobgraph for the execution.

Field Summary

Fields

Modifier and Type	Field and Description
protected Map<Integer,String>	vertexIDtoBrokerID
protected Map<Integer,Long>	vertexIDtoLoopTimeout

Constructor Summary

Constructors

Constructor and Description
StreamGraph(StreamExecutionEnvironment environment)

Method Summary

Modifier and Type	Method and Description
<IN1,IN2,OUT> void	addCoOperator(Integer vertexID, String slotSharingGroup, TwoInputStreamOperator<IN1,IN2,OUT> taskOperatorObject, TypeInformation<IN1> in1TypeInfo, TypeInformation<IN2> in2TypeInfo, TypeInformation<OUT> outTypeInfo, String operatorName)
void	addEdge(Integer upStreamVertexID, Integer downStreamVertexID, int typeNumber)
protected StreamNode	addNode(Integer vertexID, String slotSharingGroup, Class<? extends AbstractInvokable> vertexClass, StreamOperator<?> operatorObject, String operatorName)
<IN,OUT> void	addOperator(Integer vertexID, String slotSharingGroup, StreamOperator<OUT> operatorObject, TypeInformation<IN> inTypeInfo, TypeInformation<OUT> outTypeInfo, String operatorName)
<T> void	addOutputSelector(Integer vertexID, OutputSelector<T> outputSelector)
<IN,OUT> void	addSink(Integer vertexID, String slotSharingGroup, StreamOperator<OUT> operatorObject, TypeInformation<IN> inTypeInfo, TypeInformation<OUT> outTypeInfo, String operatorName)
<IN,OUT> void	addSource(Integer vertexID, String slotSharingGroup, StreamOperator<OUT> operatorObject, TypeInformation<IN> inTypeInfo, TypeInformation<OUT> outTypeInfo, String operatorName)
void	addVirtualPartitionNode(Integer originalId, Integer virtualId, StreamPartitioner<?> partitioner) Adds a new virtual node that is used to connect a downstream vertex to an input with a certain partitioning.
void	addVirtualSelectNode(Integer originalId, Integer virtualId, List<String> selectedNames) Adds a new virtual node that is used to connect a downstream vertex to only the outputs with the selected names.
void	addVirtualSideOutputNode(Integer originalId, Integer virtualId, OutputTag outputTag) Adds a new virtual node that is used to connect a downstream vertex to only the outputs with the selected side-output OutputTag.
void	clear() Remove all registered nodes etc.
Tuple2<StreamNode,StreamNode>	createIterationSourceAndSink(int loopId, int sourceId, int sinkId, long timeout, int parallelism, int maxParallelism, ResourceSpec minResources, ResourceSpec preferredResources)
void	dumpStreamingPlanAsJSON(File file)
String	getBrokerID(Integer vertexID)
CheckpointConfig	getCheckpointConfig()
StreamExecutionEnvironment	getEnvironment()
ExecutionConfig	getExecutionConfig()
Set<Tuple2<StreamNode,StreamNode>>	getIterationSourceSinkPairs()
JobGraph	getJobGraph() Gets the assembled JobGraph.
String	getJobName()
long	getLoopTimeout(Integer vertexID)
Set<Tuple2<Integer,StreamOperator<?>>>	getOperators()
Collection<Integer>	getSinkIDs()
String	getSlotSharingGroup(Integer id) Determines the slot sharing group of an operation across virtual nodes.
Collection<Integer>	getSourceIDs()
StateBackend	getStateBackend()
List<StreamEdge>	getStreamEdges(int sourceId, int targetId)
String	getStreamingPlanAsJSON()
StreamNode	getStreamNode(Integer vertexID)
Collection<StreamNode>	getStreamNodes()
protected Collection<? extends Integer>	getVertexIDs()
boolean	isChainingEnabled()
boolean	isIterative()
void	setBufferTimeout(Integer vertexID, long bufferTimeout)
void	setChaining(boolean chaining)
void	setInputFormat(Integer vertexID, InputFormat<?,?> inputFormat)
void	setJobName(String jobName)
void	setMaxParallelism(int vertexID, int maxParallelism)
void	setOneInputStateKey(Integer vertexID, KeySelector<?,?> keySelector, TypeSerializer<?> keySerializer)
<IN,OUT> void	setOperator(Integer vertexID, StreamOperator<OUT> operatorObject)
<OUT> void	setOutType(Integer vertexID, TypeInformation<OUT> outType)
void	setParallelism(Integer vertexID, int parallelism)
void	setResources(int vertexID, ResourceSpec minResources, ResourceSpec preferredResources)
void	setSerializers(Integer vertexID, TypeSerializer<?> in1, TypeSerializer<?> in2, TypeSerializer<?> out)
void	setSerializersFrom(Integer from, Integer to)
void	setStateBackend(StateBackend backend)
void	setTwoInputStateKey(Integer vertexID, KeySelector<?,?> keySelector1, KeySelector<?,?> keySelector2, TypeSerializer<?> keySerializer)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

vertexIDtoBrokerID

protected Map<Integer,String> vertexIDtoBrokerID

vertexIDtoLoopTimeout

protected Map<Integer,Long> vertexIDtoLoopTimeout

Constructor Detail

StreamGraph

public StreamGraph(StreamExecutionEnvironment environment)

Method Detail

clear

public void clear()

Remove all registered nodes etc.

getEnvironment

public StreamExecutionEnvironment getEnvironment()

getExecutionConfig

public ExecutionConfig getExecutionConfig()

getCheckpointConfig

public CheckpointConfig getCheckpointConfig()

getJobName

public String getJobName()

setJobName

public void setJobName(String jobName)

setChaining

public void setChaining(boolean chaining)

setStateBackend

public void setStateBackend(StateBackend backend)

getStateBackend

public StateBackend getStateBackend()

isChainingEnabled

public boolean isChainingEnabled()

isIterative

public boolean isIterative()

addSource

public <IN,OUT> void addSource(Integer vertexID,
                               String slotSharingGroup,
                               StreamOperator<OUT> operatorObject,
                               TypeInformation<IN> inTypeInfo,
                               TypeInformation<OUT> outTypeInfo,
                               String operatorName)

addSink

public <IN,OUT> void addSink(Integer vertexID,
                             String slotSharingGroup,
                             StreamOperator<OUT> operatorObject,
                             TypeInformation<IN> inTypeInfo,
                             TypeInformation<OUT> outTypeInfo,
                             String operatorName)

addOperator

public <IN,OUT> void addOperator(Integer vertexID,
                                 String slotSharingGroup,
                                 StreamOperator<OUT> operatorObject,
                                 TypeInformation<IN> inTypeInfo,
                                 TypeInformation<OUT> outTypeInfo,
                                 String operatorName)

addCoOperator

public <IN1,IN2,OUT> void addCoOperator(Integer vertexID,
                                        String slotSharingGroup,
                                        TwoInputStreamOperator<IN1,IN2,OUT> taskOperatorObject,
                                        TypeInformation<IN1> in1TypeInfo,
                                        TypeInformation<IN2> in2TypeInfo,
                                        TypeInformation<OUT> outTypeInfo,
                                        String operatorName)

addNode

protected StreamNode addNode(Integer vertexID,
                             String slotSharingGroup,
                             Class<? extends AbstractInvokable> vertexClass,
                             StreamOperator<?> operatorObject,
                             String operatorName)

addVirtualSelectNode

public void addVirtualSelectNode(Integer originalId,
                                 Integer virtualId,
                                 List<String> selectedNames)

Adds a new virtual node that is used to connect a downstream vertex to only the outputs with the selected names.

When adding an edge from the virtual node to a downstream node the connection will be made to the original node, only with the selected names given here.

Parameters:

originalId - ID of the node that should be connected to.

virtualId - ID of the virtual node.

selectedNames - The selected names.

addVirtualSideOutputNode

public void addVirtualSideOutputNode(Integer originalId,
                                     Integer virtualId,
                                     OutputTag outputTag)

Adds a new virtual node that is used to connect a downstream vertex to only the outputs with the selected side-output OutputTag.

Parameters:

originalId - ID of the node that should be connected to.

virtualId - ID of the virtual node.

outputTag - The selected side-output OutputTag.

addVirtualPartitionNode

public void addVirtualPartitionNode(Integer originalId,
                                    Integer virtualId,
                                    StreamPartitioner<?> partitioner)

Adds a new virtual node that is used to connect a downstream vertex to an input with a certain partitioning.

When adding an edge from the virtual node to a downstream node the connection will be made to the original node, but with the partitioning given here.

Parameters:

originalId - ID of the node that should be connected to.

virtualId - ID of the virtual node.

partitioner - The partitioner

getSlotSharingGroup

public String getSlotSharingGroup(Integer id)

Determines the slot sharing group of an operation across virtual nodes.

addEdge

public void addEdge(Integer upStreamVertexID,
                    Integer downStreamVertexID,
                    int typeNumber)

addOutputSelector

public <T> void addOutputSelector(Integer vertexID,
                                  OutputSelector<T> outputSelector)

setParallelism

public void setParallelism(Integer vertexID,
                           int parallelism)

setMaxParallelism

public void setMaxParallelism(int vertexID,
                              int maxParallelism)

setResources

public void setResources(int vertexID,
                         ResourceSpec minResources,
                         ResourceSpec preferredResources)

setOneInputStateKey

public void setOneInputStateKey(Integer vertexID,
                                KeySelector<?,?> keySelector,
                                TypeSerializer<?> keySerializer)

setTwoInputStateKey

public void setTwoInputStateKey(Integer vertexID,
                                KeySelector<?,?> keySelector1,
                                KeySelector<?,?> keySelector2,
                                TypeSerializer<?> keySerializer)

setBufferTimeout

public void setBufferTimeout(Integer vertexID,
                             long bufferTimeout)

setSerializers

public void setSerializers(Integer vertexID,
                           TypeSerializer<?> in1,
                           TypeSerializer<?> in2,
                           TypeSerializer<?> out)

setSerializersFrom

public void setSerializersFrom(Integer from,
                               Integer to)

setOutType

public <OUT> void setOutType(Integer vertexID,
                             TypeInformation<OUT> outType)

setOperator

public <IN,OUT> void setOperator(Integer vertexID,
                                 StreamOperator<OUT> operatorObject)

setInputFormat

public void setInputFormat(Integer vertexID,
                           InputFormat<?,?> inputFormat)

getStreamNode

public StreamNode getStreamNode(Integer vertexID)

getVertexIDs

protected Collection<? extends Integer> getVertexIDs()

getStreamEdges

public List<StreamEdge> getStreamEdges(int sourceId,
                                       int targetId)

getSourceIDs

public Collection<Integer> getSourceIDs()

getSinkIDs

public Collection<Integer> getSinkIDs()

getStreamNodes

public Collection<StreamNode> getStreamNodes()

getOperators

public Set<Tuple2<Integer,StreamOperator<?>>> getOperators()

getBrokerID

public String getBrokerID(Integer vertexID)

getLoopTimeout

public long getLoopTimeout(Integer vertexID)

createIterationSourceAndSink

public Tuple2<StreamNode,StreamNode> createIterationSourceAndSink(int loopId,
                                                                  int sourceId,
                                                                  int sinkId,
                                                                  long timeout,
                                                                  int parallelism,
                                                                  int maxParallelism,
                                                                  ResourceSpec minResources,
                                                                  ResourceSpec preferredResources)

getIterationSourceSinkPairs

public Set<Tuple2<StreamNode,StreamNode>> getIterationSourceSinkPairs()

getJobGraph

public JobGraph getJobGraph()

Gets the assembled JobGraph.

Specified by:

getJobGraph in class StreamingPlan

getStreamingPlanAsJSON

public String getStreamingPlanAsJSON()

Specified by:

getStreamingPlanAsJSON in class StreamingPlan

dumpStreamingPlanAsJSON

public void dumpStreamingPlanAsJSON(File file)
                             throws IOException

Specified by:

dumpStreamingPlanAsJSON in class StreamingPlan

Throws:

IOException