org.apache.flink.ml.common.broadcast.operator

Class AbstractBroadcastWrapperOperator<T,S extends org.apache.flink.streaming.api.operators.StreamOperator<T>>

java.lang.Object

org.apache.flink.ml.common.broadcast.operator.AbstractBroadcastWrapperOperator<T,S>

All Implemented Interfaces:

Serializable, org.apache.flink.api.common.state.CheckpointListener, org.apache.flink.streaming.api.operators.KeyContext, org.apache.flink.streaming.api.operators.StreamOperator<T>, org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.CheckpointedStreamOperator

Direct Known Subclasses:

OneInputBroadcastWrapperOperator, TwoInputBroadcastWrapperOperator

public abstract class AbstractBroadcastWrapperOperator<T,S extends org.apache.flink.streaming.api.operators.StreamOperator<T>>
extends Object
implements org.apache.flink.streaming.api.operators.StreamOperator<T>, org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.CheckpointedStreamOperator

Base class for the broadcast wrapper operators.

See Also:

Serialized Form

Field Summary

Fields

Modifier and Type	Field and Description
protected String[]	broadcastStreamNames variables specific for withBroadcast functionality.
protected boolean	broadcastVariablesReady whether all broadcast variables of this operator are ready.
protected org.apache.flink.streaming.runtime.tasks.StreamTask<?,?>	containingTask
protected DataCacheWriter[]	dataCacheWriters DataCacheWriter for each input.
protected boolean[]	hasPendingElements whether each input has pending elements.
protected int	indexOfSubtask index of this subtask.
protected org.apache.flink.api.common.typeinfo.TypeInformation<?>[]	inTypes type information of each input.
protected boolean[]	isBlocked whether each input is blocked.
protected org.apache.flink.api.common.operators.MailboxExecutor	mailboxExecutor
protected org.apache.flink.metrics.groups.OperatorMetricGroup	metrics
protected int	numInputs number of the inputs of this operator.
protected org.apache.flink.streaming.api.operators.StreamOperatorFactory<T>	operatorFactory
protected org.apache.flink.streaming.api.operators.Output<org.apache.flink.streaming.runtime.streamrecord.StreamRecord<T>>	output
protected org.apache.flink.streaming.api.operators.StreamOperatorParameters<T>	parameters
protected org.apache.flink.streaming.api.operators.StreamOperatorStateHandler	stateHandler
protected org.apache.flink.streaming.api.graph.StreamConfig	streamConfig
protected org.apache.flink.streaming.api.operators.InternalTimeServiceManager<?>	timeServiceManager
protected S	wrappedOperator

Method Summary

Modifier and Type	Method and Description
protected boolean	areBroadcastVariablesReady() checks whether all of broadcast variables are ready.
void	close()
protected void	endInputX(int inputIndex, org.apache.flink.util.function.ThrowingConsumer<org.apache.flink.streaming.runtime.streamrecord.StreamRecord,Exception> elementConsumer, org.apache.flink.util.function.ThrowingConsumer<org.apache.flink.streaming.api.watermark.Watermark,Exception> watermarkConsumer) extracts common processing logic in subclasses' endInput(...).
void	finish()
Object	getCurrentKey()
org.apache.flink.metrics.groups.OperatorMetricGroup	getMetricGroup()
org.apache.flink.runtime.jobgraph.OperatorID	getOperatorID()
void	initializeState(org.apache.flink.runtime.state.StateInitializationContext stateInitializationContext)
void	initializeState(org.apache.flink.streaming.api.operators.StreamTaskStateInitializer streamTaskStateManager)
void	notifyCheckpointAborted(long checkpointId)
void	notifyCheckpointComplete(long checkpointId)
void	open()
void	prepareSnapshotPreBarrier(long checkpointId)
protected void	processElementX(org.apache.flink.streaming.runtime.streamrecord.StreamRecord streamRecord, int inputIndex, org.apache.flink.util.function.ThrowingConsumer<org.apache.flink.streaming.runtime.streamrecord.StreamRecord,Exception> elementConsumer, org.apache.flink.util.function.ThrowingConsumer<org.apache.flink.streaming.api.watermark.Watermark,Exception> watermarkConsumer) extracts common processing logic in subclasses' processing elements.
protected void	processWatermarkX(org.apache.flink.streaming.api.watermark.Watermark watermark, int inputIndex, org.apache.flink.util.function.ThrowingConsumer<org.apache.flink.streaming.runtime.streamrecord.StreamRecord,Exception> elementConsumer, org.apache.flink.util.function.ThrowingConsumer<org.apache.flink.streaming.api.watermark.Watermark,Exception> watermarkConsumer) extracts common processing logic in subclasses' processing watermarks.
void	setCurrentKey(Object key)
void	setKeyContextElement1(org.apache.flink.streaming.runtime.streamrecord.StreamRecord<?> record)
void	setKeyContextElement2(org.apache.flink.streaming.runtime.streamrecord.StreamRecord<?> record)
org.apache.flink.streaming.api.operators.OperatorSnapshotFutures	snapshotState(long checkpointId, long timestamp, org.apache.flink.runtime.checkpoint.CheckpointOptions checkpointOptions, org.apache.flink.runtime.state.CheckpointStreamFactory storageLocation)
void	snapshotState(org.apache.flink.runtime.state.StateSnapshotContext stateSnapshotContext)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

parameters

protected final org.apache.flink.streaming.api.operators.StreamOperatorParameters<T> parameters

streamConfig

protected final org.apache.flink.streaming.api.graph.StreamConfig streamConfig

containingTask

protected final org.apache.flink.streaming.runtime.tasks.StreamTask<?,?> containingTask

output

protected final org.apache.flink.streaming.api.operators.Output<org.apache.flink.streaming.runtime.streamrecord.StreamRecord<T>> output

operatorFactory

protected final org.apache.flink.streaming.api.operators.StreamOperatorFactory<T> operatorFactory

metrics

protected final org.apache.flink.metrics.groups.OperatorMetricGroup metrics

wrappedOperator

protected final S extends org.apache.flink.streaming.api.operators.StreamOperator<T> wrappedOperator

stateHandler

protected transient org.apache.flink.streaming.api.operators.StreamOperatorStateHandler stateHandler

timeServiceManager

protected transient org.apache.flink.streaming.api.operators.InternalTimeServiceManager<?> timeServiceManager

mailboxExecutor

protected final org.apache.flink.api.common.operators.MailboxExecutor mailboxExecutor

broadcastStreamNames

protected final String[] broadcastStreamNames

variables specific for withBroadcast functionality.

isBlocked

protected final boolean[] isBlocked

whether each input is blocked. Inputs with broadcast variables can only process their input records after broadcast variables are ready. One input is non-blocked if it can consume its inputs (by caching) when broadcast variables are not ready. Otherwise it has to block the processing and wait until the broadcast variables are ready to be accessed.

inTypes

protected final org.apache.flink.api.common.typeinfo.TypeInformation<?>[] inTypes

type information of each input.

broadcastVariablesReady

protected boolean broadcastVariablesReady

whether all broadcast variables of this operator are ready.

indexOfSubtask

protected final transient int indexOfSubtask

index of this subtask.

numInputs

protected final int numInputs

number of the inputs of this operator.

dataCacheWriters

protected DataCacheWriter[] dataCacheWriters

DataCacheWriter for each input.

hasPendingElements

protected boolean[] hasPendingElements

whether each input has pending elements.

Method Detail

areBroadcastVariablesReady

protected boolean areBroadcastVariablesReady()

checks whether all of broadcast variables are ready. Besides it maintains a state {broadcastVariablesReady} to avoiding invoking BroadcastContext.isCacheFinished(...) repeatedly. Finally, it sets broadcast variables for {wrappedOperatorRuntimeContext} if the broadcast variables are ready.

Returns:

true if all broadcast variables are ready, false otherwise.

processElementX

protected void processElementX(org.apache.flink.streaming.runtime.streamrecord.StreamRecord streamRecord,
                               int inputIndex,
                               org.apache.flink.util.function.ThrowingConsumer<org.apache.flink.streaming.runtime.streamrecord.StreamRecord,Exception> elementConsumer,
                               org.apache.flink.util.function.ThrowingConsumer<org.apache.flink.streaming.api.watermark.Watermark,Exception> watermarkConsumer)
                        throws Exception

extracts common processing logic in subclasses' processing elements.

Parameters:

streamRecord - the input record.

inputIndex - input id, starts from zero.

elementConsumer - the consumer function of StreamRecord, i.e., operator.processElement(...).

watermarkConsumer - the consumer function of WaterMark, i.e., operator.processWatermark(...).

Throws:

Exception - possible exception.

processWatermarkX

protected void processWatermarkX(org.apache.flink.streaming.api.watermark.Watermark watermark,
                                 int inputIndex,
                                 org.apache.flink.util.function.ThrowingConsumer<org.apache.flink.streaming.runtime.streamrecord.StreamRecord,Exception> elementConsumer,
                                 org.apache.flink.util.function.ThrowingConsumer<org.apache.flink.streaming.api.watermark.Watermark,Exception> watermarkConsumer)
                          throws Exception

extracts common processing logic in subclasses' processing watermarks.

Parameters:

watermark - the input watermark.

inputIndex - input id, starts from zero.

elementConsumer - the consumer function of StreamRecord, i.e., operator.processElement(...).

watermarkConsumer - the consumer function of WaterMark, i.e., operator.processWatermark(...).

Throws:

Exception - possible exception.

endInputX

protected void endInputX(int inputIndex,
                         org.apache.flink.util.function.ThrowingConsumer<org.apache.flink.streaming.runtime.streamrecord.StreamRecord,Exception> elementConsumer,
                         org.apache.flink.util.function.ThrowingConsumer<org.apache.flink.streaming.api.watermark.Watermark,Exception> watermarkConsumer)
                  throws Exception

extracts common processing logic in subclasses' endInput(...).

Parameters:

inputIndex - input id, starts from zero.

elementConsumer - the consumer function of StreamRecord, i.e., operator.processElement(...).

watermarkConsumer - the consumer function of WaterMark, i.e., operator.processWatermark(...).

Throws:

Exception - possible exception.

open

public void open()
          throws Exception

Specified by:

open in interface org.apache.flink.streaming.api.operators.StreamOperator<T>

Throws:

Exception

close

public void close()
           throws Exception

Specified by:

close in interface org.apache.flink.streaming.api.operators.StreamOperator<T>

Throws:

Exception

finish

public void finish()
            throws Exception

Specified by:

finish in interface org.apache.flink.streaming.api.operators.StreamOperator<T>

Throws:

Exception

prepareSnapshotPreBarrier

public void prepareSnapshotPreBarrier(long checkpointId)
                               throws Exception

Specified by:

prepareSnapshotPreBarrier in interface org.apache.flink.streaming.api.operators.StreamOperator<T>

Throws:

Exception

initializeState

public void initializeState(org.apache.flink.streaming.api.operators.StreamTaskStateInitializer streamTaskStateManager)
                     throws Exception

Specified by:

initializeState in interface org.apache.flink.streaming.api.operators.StreamOperator<T>

Throws:

Exception

snapshotState

public org.apache.flink.streaming.api.operators.OperatorSnapshotFutures snapshotState(long checkpointId,
                                                                                      long timestamp,
                                                                                      org.apache.flink.runtime.checkpoint.CheckpointOptions checkpointOptions,
                                                                                      org.apache.flink.runtime.state.CheckpointStreamFactory storageLocation)
                                                                               throws Exception

Specified by:

snapshotState in interface org.apache.flink.streaming.api.operators.StreamOperator<T>

Throws:

Exception

initializeState

public void initializeState(org.apache.flink.runtime.state.StateInitializationContext stateInitializationContext)
                     throws Exception

Specified by:

initializeState in interface org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.CheckpointedStreamOperator

Throws:

Exception

snapshotState

public void snapshotState(org.apache.flink.runtime.state.StateSnapshotContext stateSnapshotContext)
                   throws Exception

Specified by:

snapshotState in interface org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.CheckpointedStreamOperator

Throws:

Exception

setKeyContextElement1

public void setKeyContextElement1(org.apache.flink.streaming.runtime.streamrecord.StreamRecord<?> record)
                           throws Exception

Specified by:

setKeyContextElement1 in interface org.apache.flink.streaming.api.operators.StreamOperator<T>

Throws:

Exception

setKeyContextElement2

public void setKeyContextElement2(org.apache.flink.streaming.runtime.streamrecord.StreamRecord<?> record)
                           throws Exception

Specified by:

setKeyContextElement2 in interface org.apache.flink.streaming.api.operators.StreamOperator<T>

Throws:

Exception

getMetricGroup

public org.apache.flink.metrics.groups.OperatorMetricGroup getMetricGroup()

Specified by:

getMetricGroup in interface org.apache.flink.streaming.api.operators.StreamOperator<T>

getOperatorID

public org.apache.flink.runtime.jobgraph.OperatorID getOperatorID()

Specified by:

getOperatorID in interface org.apache.flink.streaming.api.operators.StreamOperator<T>

notifyCheckpointComplete

public void notifyCheckpointComplete(long checkpointId)
                              throws Exception

Specified by:

notifyCheckpointComplete in interface org.apache.flink.api.common.state.CheckpointListener

Throws:

Exception

notifyCheckpointAborted

public void notifyCheckpointAborted(long checkpointId)
                             throws Exception

Specified by:

notifyCheckpointAborted in interface org.apache.flink.api.common.state.CheckpointListener

Throws:

Exception

setCurrentKey

public void setCurrentKey(Object key)

Specified by:

setCurrentKey in interface org.apache.flink.streaming.api.operators.KeyContext

getCurrentKey

public Object getCurrentKey()

Specified by:

getCurrentKey in interface org.apache.flink.streaming.api.operators.KeyContext