org.apache.flink.streaming.api.graph

Class StreamGraph

java.lang.Object

org.apache.flink.streaming.api.graph.StreamGraph

All Implemented Interfaces:

Pipeline

@Internal
public class StreamGraph
extends Object
implements Pipeline

Class representing the streaming topology. It contains all the information necessary to build the jobgraph for the execution.

Field Summary

Fields

Modifier and Type	Field and Description
static String	ITERATION_SINK_NAME_PREFIX
static String	ITERATION_SOURCE_NAME_PREFIX
protected Map<Integer,String>	vertexIDtoBrokerID
protected Map<Integer,Long>	vertexIDtoLoopTimeout

Constructor Summary

Constructors

Constructor and Description
StreamGraph(Configuration jobConfiguration, ExecutionConfig executionConfig, CheckpointConfig checkpointConfig, SavepointRestoreSettings savepointRestoreSettings)

Method Summary

Modifier and Type	Method and Description
<IN1,IN2,OUT> void	addCoOperator(Integer vertexID, String slotSharingGroup, String coLocationGroup, StreamOperatorFactory<OUT> taskOperatorFactory, TypeInformation<IN1> in1TypeInfo, TypeInformation<IN2> in2TypeInfo, TypeInformation<OUT> outTypeInfo, String operatorName)
void	addEdge(Integer upStreamVertexID, Integer downStreamVertexID, int typeNumber)
void	addEdge(Integer upStreamVertexID, Integer downStreamVertexID, int typeNumber, IntermediateDataSetID intermediateDataSetId)
<IN,OUT> void	addLegacySource(Integer vertexID, String slotSharingGroup, String coLocationGroup, StreamOperatorFactory<OUT> operatorFactory, TypeInformation<IN> inTypeInfo, TypeInformation<OUT> outTypeInfo, String operatorName)
<OUT> void	addMultipleInputOperator(Integer vertexID, String slotSharingGroup, String coLocationGroup, StreamOperatorFactory<OUT> operatorFactory, List<TypeInformation<?>> inTypeInfos, TypeInformation<OUT> outTypeInfo, String operatorName)
protected StreamNode	addNode(Integer vertexID, String slotSharingGroup, String coLocationGroup, Class<? extends TaskInvokable> vertexClass, StreamOperatorFactory<?> operatorFactory, String operatorName)
<IN,OUT> void	addOperator(Integer vertexID, String slotSharingGroup, String coLocationGroup, StreamOperatorFactory<OUT> operatorFactory, TypeInformation<IN> inTypeInfo, TypeInformation<OUT> outTypeInfo, String operatorName)
<IN,OUT> void	addSink(Integer vertexID, String slotSharingGroup, String coLocationGroup, StreamOperatorFactory<OUT> operatorFactory, TypeInformation<IN> inTypeInfo, TypeInformation<OUT> outTypeInfo, String operatorName)
<IN,OUT> void	addSource(Integer vertexID, String slotSharingGroup, String coLocationGroup, SourceOperatorFactory<OUT> operatorFactory, TypeInformation<IN> inTypeInfo, TypeInformation<OUT> outTypeInfo, String operatorName)
void	addVirtualPartitionNode(Integer originalId, Integer virtualId, StreamPartitioner<?> partitioner, StreamExchangeMode exchangeMode) Adds a new virtual node that is used to connect a downstream vertex to an input with a certain partitioning.
void	addVirtualSideOutputNode(Integer originalId, Integer virtualId, OutputTag outputTag) Adds a new virtual node that is used to connect a downstream vertex to only the outputs with the selected side-output OutputTag.
void	clear() Remove all registered nodes etc.
Tuple2<StreamNode,StreamNode>	createIterationSourceAndSink(int loopId, int sourceId, int sinkId, long timeout, int parallelism, int maxParallelism, ResourceSpec minResources, ResourceSpec preferredResources)
String	getBrokerID(Integer vertexID)
CheckpointConfig	getCheckpointConfig()
CheckpointStorage	getCheckpointStorage()
ExecutionConfig	getExecutionConfig()
GlobalStreamExchangeMode	getGlobalStreamExchangeMode()
Set<Tuple2<StreamNode,StreamNode>>	getIterationSourceSinkPairs()
Configuration	getJobConfiguration()
JobGraph	getJobGraph() Gets the assembled JobGraph with a random JobID.
JobGraph	getJobGraph(ClassLoader userClassLoader, JobID jobID) Gets the assembled JobGraph with a specified JobID.
String	getJobName()
List<JobStatusHook>	getJobStatusHooks()
JobType	getJobType()
long	getLoopTimeout(Integer vertexID)
SavepointRestoreSettings	getSavepointRestoreSettings()
Collection<Integer>	getSinkIDs()
String	getSlotSharingGroup(Integer id) Determines the slot sharing group of an operation across virtual nodes.
Optional<ResourceProfile>	getSlotSharingGroupResource(String groupId)
Collection<Integer>	getSourceIDs()
StreamNode	getSourceVertex(StreamEdge edge)
StateBackend	getStateBackend()
List<StreamEdge>	getStreamEdges(int sourceId)
List<StreamEdge>	getStreamEdges(int sourceId, int targetId)
List<StreamEdge>	getStreamEdgesOrThrow(int sourceId, int targetId) Deprecated.
String	getStreamingPlanAsJSON()
StreamNode	getStreamNode(Integer vertexID)
Collection<StreamNode>	getStreamNodes()
StreamNode	getTargetVertex(StreamEdge edge)
TimeCharacteristic	getTimeCharacteristic()
InternalTimeServiceManager.Provider	getTimerServiceProvider()
Collection<Tuple2<String,DistributedCache.DistributedCacheEntry>>	getUserArtifacts()
PipelineOptions.VertexDescriptionMode	getVertexDescriptionMode()
protected Collection<? extends Integer>	getVertexIDs()
boolean	hasFineGrainedResource()
boolean	isAllVerticesInSameSlotSharingGroupByDefault() Gets whether to put all vertices into the same slot sharing group by default.
boolean	isAutoParallelismEnabled()
boolean	isChainingEnabled()
boolean	isChainingOfOperatorsWithDifferentMaxParallelismEnabled()
boolean	isDynamic()
boolean	isEnableCheckpointsAfterTasksFinish()
boolean	isIterative()
boolean	isVertexNameIncludeIndexPrefix()
void	registerJobStatusHook(JobStatusHook hook) Registers the JobStatusHook.
void	setAllVerticesInSameSlotSharingGroupByDefault(boolean allVerticesInSameSlotSharingGroupByDefault) Set whether to put all vertices into the same slot sharing group by default.
void	setAutoParallelismEnabled(boolean autoParallelismEnabled)
void	setBufferTimeout(Integer vertexID, long bufferTimeout)
void	setCheckpointStorage(CheckpointStorage checkpointStorage)
void	setDynamic(boolean dynamic)
void	setEnableCheckpointsAfterTasksFinish(boolean enableCheckpointsAfterTasksFinish)
void	setGlobalStreamExchangeMode(GlobalStreamExchangeMode globalExchangeMode)
void	setInputFormat(Integer vertexID, InputFormat<?,?> inputFormat)
void	setJobName(String jobName)
void	setJobType(JobType jobType)
void	setManagedMemoryUseCaseWeights(int vertexID, Map<ManagedMemoryUseCase,Integer> operatorScopeUseCaseWeights, Set<ManagedMemoryUseCase> slotScopeUseCases)
void	setMaxParallelism(int vertexID, int maxParallelism)
void	setMultipleInputStateKey(Integer vertexID, List<KeySelector<?,?>> keySelectors, TypeSerializer<?> keySerializer)
void	setOneInputStateKey(Integer vertexID, KeySelector<?,?> keySelector, TypeSerializer<?> keySerializer)
void	setOutputFormat(Integer vertexID, OutputFormat<?> outputFormat)
void	setParallelism(Integer vertexID, int parallelism)
void	setParallelism(Integer vertexId, int parallelism, boolean parallelismConfigured)
void	setResources(int vertexID, ResourceSpec minResources, ResourceSpec preferredResources)
void	setSavepointRestoreSettings(SavepointRestoreSettings savepointRestoreSettings)
void	setSerializers(Integer vertexID, TypeSerializer<?> in1, TypeSerializer<?> in2, TypeSerializer<?> out)
void	setSlotSharingGroupResource(Map<String,ResourceProfile> slotSharingGroupResources)
void	setStateBackend(StateBackend backend)
void	setSupportsConcurrentExecutionAttempts(Integer vertexId, boolean supportsConcurrentExecutionAttempts)
void	setTimeCharacteristic(TimeCharacteristic timeCharacteristic)
void	setTimerServiceProvider(InternalTimeServiceManager.Provider timerServiceProvider)
void	setTransformationUID(Integer nodeId, String transformationId)
void	setTwoInputStateKey(Integer vertexID, KeySelector<?,?> keySelector1, KeySelector<?,?> keySelector2, TypeSerializer<?> keySerializer)
void	setVertexDescriptionMode(PipelineOptions.VertexDescriptionMode mode)
void	setVertexNameIncludeIndexPrefix(boolean includePrefix)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

ITERATION_SOURCE_NAME_PREFIX

public static final String ITERATION_SOURCE_NAME_PREFIX

See Also:

Constant Field Values

ITERATION_SINK_NAME_PREFIX

public static final String ITERATION_SINK_NAME_PREFIX

See Also:

Constant Field Values

vertexIDtoBrokerID

protected Map<Integer,String> vertexIDtoBrokerID

vertexIDtoLoopTimeout

protected Map<Integer,Long> vertexIDtoLoopTimeout

Constructor Detail

StreamGraph

public StreamGraph(Configuration jobConfiguration,
                   ExecutionConfig executionConfig,
                   CheckpointConfig checkpointConfig,
                   SavepointRestoreSettings savepointRestoreSettings)

Method Detail

clear

public void clear()

Remove all registered nodes etc.

getExecutionConfig

public ExecutionConfig getExecutionConfig()

getJobConfiguration

public Configuration getJobConfiguration()

getCheckpointConfig

public CheckpointConfig getCheckpointConfig()

setSavepointRestoreSettings

public void setSavepointRestoreSettings(SavepointRestoreSettings savepointRestoreSettings)

getSavepointRestoreSettings

public SavepointRestoreSettings getSavepointRestoreSettings()

getJobName

public String getJobName()

setJobName

public void setJobName(String jobName)

setStateBackend

public void setStateBackend(StateBackend backend)

getStateBackend

public StateBackend getStateBackend()

setCheckpointStorage

public void setCheckpointStorage(CheckpointStorage checkpointStorage)

getCheckpointStorage

public CheckpointStorage getCheckpointStorage()

getTimerServiceProvider

public InternalTimeServiceManager.Provider getTimerServiceProvider()

setTimerServiceProvider

public void setTimerServiceProvider(InternalTimeServiceManager.Provider timerServiceProvider)

getUserArtifacts

public Collection<Tuple2<String,DistributedCache.DistributedCacheEntry>> getUserArtifacts()

getTimeCharacteristic

public TimeCharacteristic getTimeCharacteristic()

setTimeCharacteristic

public void setTimeCharacteristic(TimeCharacteristic timeCharacteristic)

getGlobalStreamExchangeMode

public GlobalStreamExchangeMode getGlobalStreamExchangeMode()

setGlobalStreamExchangeMode

public void setGlobalStreamExchangeMode(GlobalStreamExchangeMode globalExchangeMode)

setSlotSharingGroupResource

public void setSlotSharingGroupResource(Map<String,ResourceProfile> slotSharingGroupResources)

getSlotSharingGroupResource

public Optional<ResourceProfile> getSlotSharingGroupResource(String groupId)

hasFineGrainedResource

public boolean hasFineGrainedResource()

setAllVerticesInSameSlotSharingGroupByDefault

public void setAllVerticesInSameSlotSharingGroupByDefault(boolean allVerticesInSameSlotSharingGroupByDefault)

Set whether to put all vertices into the same slot sharing group by default.

Parameters:

allVerticesInSameSlotSharingGroupByDefault - indicates whether to put all vertices into the same slot sharing group by default.

isAllVerticesInSameSlotSharingGroupByDefault

public boolean isAllVerticesInSameSlotSharingGroupByDefault()

Gets whether to put all vertices into the same slot sharing group by default.

Returns:

whether to put all vertices into the same slot sharing group by default.

isEnableCheckpointsAfterTasksFinish

public boolean isEnableCheckpointsAfterTasksFinish()

setEnableCheckpointsAfterTasksFinish

public void setEnableCheckpointsAfterTasksFinish(boolean enableCheckpointsAfterTasksFinish)

isChainingEnabled

public boolean isChainingEnabled()

isChainingOfOperatorsWithDifferentMaxParallelismEnabled

public boolean isChainingOfOperatorsWithDifferentMaxParallelismEnabled()

isIterative

public boolean isIterative()

addSource

public <IN,OUT> void addSource(Integer vertexID,
                               @Nullable
                               String slotSharingGroup,
                               @Nullable
                               String coLocationGroup,
                               SourceOperatorFactory<OUT> operatorFactory,
                               TypeInformation<IN> inTypeInfo,
                               TypeInformation<OUT> outTypeInfo,
                               String operatorName)

addLegacySource

public <IN,OUT> void addLegacySource(Integer vertexID,
                                     @Nullable
                                     String slotSharingGroup,
                                     @Nullable
                                     String coLocationGroup,
                                     StreamOperatorFactory<OUT> operatorFactory,
                                     TypeInformation<IN> inTypeInfo,
                                     TypeInformation<OUT> outTypeInfo,
                                     String operatorName)

addSink

public <IN,OUT> void addSink(Integer vertexID,
                             @Nullable
                             String slotSharingGroup,
                             @Nullable
                             String coLocationGroup,
                             StreamOperatorFactory<OUT> operatorFactory,
                             TypeInformation<IN> inTypeInfo,
                             TypeInformation<OUT> outTypeInfo,
                             String operatorName)

addOperator

public <IN,OUT> void addOperator(Integer vertexID,
                                 @Nullable
                                 String slotSharingGroup,
                                 @Nullable
                                 String coLocationGroup,
                                 StreamOperatorFactory<OUT> operatorFactory,
                                 TypeInformation<IN> inTypeInfo,
                                 TypeInformation<OUT> outTypeInfo,
                                 String operatorName)

addCoOperator

public <IN1,IN2,OUT> void addCoOperator(Integer vertexID,
                                        String slotSharingGroup,
                                        @Nullable
                                        String coLocationGroup,
                                        StreamOperatorFactory<OUT> taskOperatorFactory,
                                        TypeInformation<IN1> in1TypeInfo,
                                        TypeInformation<IN2> in2TypeInfo,
                                        TypeInformation<OUT> outTypeInfo,
                                        String operatorName)

addMultipleInputOperator

public <OUT> void addMultipleInputOperator(Integer vertexID,
                                           String slotSharingGroup,
                                           @Nullable
                                           String coLocationGroup,
                                           StreamOperatorFactory<OUT> operatorFactory,
                                           List<TypeInformation<?>> inTypeInfos,
                                           TypeInformation<OUT> outTypeInfo,
                                           String operatorName)

addNode

protected StreamNode addNode(Integer vertexID,
                             @Nullable
                             String slotSharingGroup,
                             @Nullable
                             String coLocationGroup,
                             Class<? extends TaskInvokable> vertexClass,
                             @Nullable
                             StreamOperatorFactory<?> operatorFactory,
                             String operatorName)

addVirtualSideOutputNode

public void addVirtualSideOutputNode(Integer originalId,
                                     Integer virtualId,
                                     OutputTag outputTag)

Adds a new virtual node that is used to connect a downstream vertex to only the outputs with the selected side-output OutputTag.

Parameters:

originalId - ID of the node that should be connected to.

virtualId - ID of the virtual node.

outputTag - The selected side-output OutputTag.

addVirtualPartitionNode

public void addVirtualPartitionNode(Integer originalId,
                                    Integer virtualId,
                                    StreamPartitioner<?> partitioner,
                                    StreamExchangeMode exchangeMode)

Adds a new virtual node that is used to connect a downstream vertex to an input with a certain partitioning.

When adding an edge from the virtual node to a downstream node the connection will be made to the original node, but with the partitioning given here.

Parameters:

originalId - ID of the node that should be connected to.

virtualId - ID of the virtual node.

partitioner - The partitioner

getSlotSharingGroup

public String getSlotSharingGroup(Integer id)

Determines the slot sharing group of an operation across virtual nodes.

addEdge

public void addEdge(Integer upStreamVertexID,
                    Integer downStreamVertexID,
                    int typeNumber)

addEdge

public void addEdge(Integer upStreamVertexID,
                    Integer downStreamVertexID,
                    int typeNumber,
                    IntermediateDataSetID intermediateDataSetId)

setParallelism

public void setParallelism(Integer vertexID,
                           int parallelism)

isDynamic

public boolean isDynamic()

setParallelism

public void setParallelism(Integer vertexId,
                           int parallelism,
                           boolean parallelismConfigured)

setDynamic

public void setDynamic(boolean dynamic)

setMaxParallelism

public void setMaxParallelism(int vertexID,
                              int maxParallelism)

setResources

public void setResources(int vertexID,
                         ResourceSpec minResources,
                         ResourceSpec preferredResources)

setManagedMemoryUseCaseWeights

public void setManagedMemoryUseCaseWeights(int vertexID,
                                           Map<ManagedMemoryUseCase,Integer> operatorScopeUseCaseWeights,
                                           Set<ManagedMemoryUseCase> slotScopeUseCases)

setOneInputStateKey

public void setOneInputStateKey(Integer vertexID,
                                KeySelector<?,?> keySelector,
                                TypeSerializer<?> keySerializer)

setTwoInputStateKey

public void setTwoInputStateKey(Integer vertexID,
                                KeySelector<?,?> keySelector1,
                                KeySelector<?,?> keySelector2,
                                TypeSerializer<?> keySerializer)

setMultipleInputStateKey

public void setMultipleInputStateKey(Integer vertexID,
                                     List<KeySelector<?,?>> keySelectors,
                                     TypeSerializer<?> keySerializer)

setBufferTimeout

public void setBufferTimeout(Integer vertexID,
                             long bufferTimeout)

setSerializers

public void setSerializers(Integer vertexID,
                           TypeSerializer<?> in1,
                           TypeSerializer<?> in2,
                           TypeSerializer<?> out)

setInputFormat

public void setInputFormat(Integer vertexID,
                           InputFormat<?,?> inputFormat)

setOutputFormat

public void setOutputFormat(Integer vertexID,
                            OutputFormat<?> outputFormat)

setTransformationUID

public void setTransformationUID(Integer nodeId,
                                 String transformationId)

getStreamNode

public StreamNode getStreamNode(Integer vertexID)

getVertexIDs

protected Collection<? extends Integer> getVertexIDs()

getStreamEdges

@VisibleForTesting
public List<StreamEdge> getStreamEdges(int sourceId)

getStreamEdges

@VisibleForTesting
public List<StreamEdge> getStreamEdges(int sourceId,
                                                          int targetId)

getStreamEdgesOrThrow

@VisibleForTesting
 @Deprecated
public List<StreamEdge> getStreamEdgesOrThrow(int sourceId,
                                                                              int targetId)

Deprecated.

getSourceIDs

public Collection<Integer> getSourceIDs()

getSinkIDs

public Collection<Integer> getSinkIDs()

getStreamNodes

public Collection<StreamNode> getStreamNodes()

getBrokerID

public String getBrokerID(Integer vertexID)

getLoopTimeout

public long getLoopTimeout(Integer vertexID)

createIterationSourceAndSink

public Tuple2<StreamNode,StreamNode> createIterationSourceAndSink(int loopId,
                                                                  int sourceId,
                                                                  int sinkId,
                                                                  long timeout,
                                                                  int parallelism,
                                                                  int maxParallelism,
                                                                  ResourceSpec minResources,
                                                                  ResourceSpec preferredResources)

getIterationSourceSinkPairs

public Set<Tuple2<StreamNode,StreamNode>> getIterationSourceSinkPairs()

getSourceVertex

public StreamNode getSourceVertex(StreamEdge edge)

getTargetVertex

public StreamNode getTargetVertex(StreamEdge edge)

getJobGraph

@VisibleForTesting
public JobGraph getJobGraph()

Gets the assembled JobGraph with a random JobID.

getJobGraph

public JobGraph getJobGraph(ClassLoader userClassLoader,
                            @Nullable
                            JobID jobID)

Gets the assembled JobGraph with a specified JobID.

getStreamingPlanAsJSON

public String getStreamingPlanAsJSON()

setJobType

public void setJobType(JobType jobType)

getJobType

public JobType getJobType()

isAutoParallelismEnabled

public boolean isAutoParallelismEnabled()

setAutoParallelismEnabled

public void setAutoParallelismEnabled(boolean autoParallelismEnabled)

getVertexDescriptionMode

public PipelineOptions.VertexDescriptionMode getVertexDescriptionMode()

setVertexDescriptionMode

public void setVertexDescriptionMode(PipelineOptions.VertexDescriptionMode mode)

setVertexNameIncludeIndexPrefix

public void setVertexNameIncludeIndexPrefix(boolean includePrefix)

isVertexNameIncludeIndexPrefix

public boolean isVertexNameIncludeIndexPrefix()

registerJobStatusHook

public void registerJobStatusHook(JobStatusHook hook)

Registers the JobStatusHook.

getJobStatusHooks

public List<JobStatusHook> getJobStatusHooks()

setSupportsConcurrentExecutionAttempts

public void setSupportsConcurrentExecutionAttempts(Integer vertexId,
                                                   boolean supportsConcurrentExecutionAttempts)