@Internal public class StreamingFileWriter<IN> extends AbstractStreamingWriter<IN,PartitionCommitInfo>
PartitionCommitInfo
to downstream.buckets, currentWatermark
chainingStrategy, config, latencyStats, LOG, metrics, output, processingTimeService
Constructor and Description |
---|
StreamingFileWriter(long bucketCheckInterval,
StreamingFileSink.BucketsBuilder<IN,String,? extends StreamingFileSink.BucketsBuilder<IN,String,?>> bucketsBuilder,
List<String> partitionKeys,
Configuration conf) |
Modifier and Type | Method and Description |
---|---|
protected void |
commitUpToCheckpoint(long checkpointId)
Commit up to this checkpoint id.
|
void |
initializeState(StateInitializationContext context)
Stream operators with state which can be restored need to override this hook method.
|
protected void |
onPartFileOpened(String s,
Path newPath)
Notifies a new file has been opened.
|
protected void |
partitionCreated(String partition)
Notifies a partition created.
|
protected void |
partitionInactive(String partition)
Notifies a partition become inactive.
|
void |
snapshotState(StateSnapshotContext context)
Stream operators with state, which want to participate in a snapshot need to override this
hook method.
|
close, endInput, notifyCheckpointComplete, processElement, processWatermark
finish, getChainingStrategy, getContainingTask, getCurrentKey, getExecutionConfig, getInternalTimerService, getKeyedStateBackend, getKeyedStateStore, getMetricGroup, getOperatorConfig, getOperatorID, getOperatorName, getOperatorStateBackend, getOrCreateKeyedState, getPartitionedState, getPartitionedState, getProcessingTimeService, getRuntimeContext, getTimeServiceManager, getUserCodeClassloader, hasKeyContext1, hasKeyContext2, initializeState, isUsingCustomRawKeyedState, notifyCheckpointAborted, open, prepareSnapshotPreBarrier, processLatencyMarker, processLatencyMarker1, processLatencyMarker2, processWatermark1, processWatermark2, processWatermarkStatus, processWatermarkStatus1, processWatermarkStatus2, reportOrForwardLatencyMarker, setChainingStrategy, setCurrentKey, setKeyContextElement1, setKeyContextElement2, setProcessingTimeService, setup, snapshotState
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
setKeyContextElement
finish, getMetricGroup, getOperatorID, initializeState, open, prepareSnapshotPreBarrier, setKeyContextElement1, setKeyContextElement2, snapshotState
notifyCheckpointAborted
getCurrentKey, setCurrentKey
processLatencyMarker, processWatermarkStatus
hasKeyContext
public StreamingFileWriter(long bucketCheckInterval, StreamingFileSink.BucketsBuilder<IN,String,? extends StreamingFileSink.BucketsBuilder<IN,String,?>> bucketsBuilder, List<String> partitionKeys, Configuration conf)
public void initializeState(StateInitializationContext context) throws Exception
AbstractStreamOperator
initializeState
in interface StreamOperatorStateHandler.CheckpointedStreamOperator
initializeState
in class AbstractStreamingWriter<IN,PartitionCommitInfo>
context
- context that allows to register different states.Exception
protected void partitionCreated(String partition)
AbstractStreamingWriter
partitionCreated
in class AbstractStreamingWriter<IN,PartitionCommitInfo>
protected void partitionInactive(String partition)
AbstractStreamingWriter
partitionInactive
in class AbstractStreamingWriter<IN,PartitionCommitInfo>
protected void onPartFileOpened(String s, Path newPath)
AbstractStreamingWriter
Note that this does not mean that the file has been created in the file system. It is only created logically and the actual file will be generated after it is committed.
onPartFileOpened
in class AbstractStreamingWriter<IN,PartitionCommitInfo>
public void snapshotState(StateSnapshotContext context) throws Exception
AbstractStreamOperator
snapshotState
in interface StreamOperatorStateHandler.CheckpointedStreamOperator
snapshotState
in class AbstractStreamingWriter<IN,PartitionCommitInfo>
context
- context that provides information and means required for taking a snapshotException
protected void commitUpToCheckpoint(long checkpointId) throws Exception
AbstractStreamingWriter
commitUpToCheckpoint
in class AbstractStreamingWriter<IN,PartitionCommitInfo>
Exception
Copyright © 2014–2024 The Apache Software Foundation. All rights reserved.