org.apache.flink.api.common.io.CheckpointableInputFormat Java Examples

The following examples show how to use org.apache.flink.api.common.io.CheckpointableInputFormat. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ContinuousFileReaderOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void loadSplit(T split) throws IOException {
	Preconditions.checkState(state != ReaderState.READING && state != ReaderState.CLOSED, "can't load split in state %s", state);
	Preconditions.checkNotNull(split, "split is null");
	LOG.debug("load split: {}", split);
	currentSplit = split;
	if (this.format instanceof RichInputFormat) {
		((RichInputFormat) this.format).openInputFormat();
	}
	if (format instanceof CheckpointableInputFormat && currentSplit.getSplitState() != null) {
		// recovering after a node failure with an input
		// format that supports resetting the offset
		((CheckpointableInputFormat<T, Serializable>) format).
				reopen(currentSplit, currentSplit.getSplitState());
	} else {
		// we either have a new split, or we recovered from a node
		// failure but the input format does not support resetting the offset.
		format.open(currentSplit);
	}

	// reset the restored state to null for the next iteration
	currentSplit.resetSplitState();
}
Example #2
Source File: ContinuousFileReaderOperator.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private List<TimestampedFileInputSplit> getReaderState() throws IOException {
	List<TimestampedFileInputSplit> snapshot = new ArrayList<>(this.pendingSplits.size());
	if (currentSplit != null) {
		if (this.format instanceof CheckpointableInputFormat && this.isSplitOpen) {
			Serializable formatState =
				((CheckpointableInputFormat<TimestampedFileInputSplit, Serializable>) this.format).getCurrentState();
			this.currentSplit.setSplitState(formatState);
		}
		snapshot.add(this.currentSplit);
	}
	snapshot.addAll(this.pendingSplits);
	return snapshot;
}
Example #3
Source File: ContinuousFileReaderOperator.java    From flink with Apache License 2.0 5 votes vote down vote up 
private List<TimestampedFileInputSplit> getReaderState() throws IOException {
	List<TimestampedFileInputSplit> snapshot = new ArrayList<>(this.pendingSplits.size());
	if (currentSplit != null) {
		if (this.format instanceof CheckpointableInputFormat && this.isSplitOpen) {
			Serializable formatState =
				((CheckpointableInputFormat<TimestampedFileInputSplit, Serializable>) this.format).getCurrentState();
			this.currentSplit.setSplitState(formatState);
		}
		snapshot.add(this.currentSplit);
	}
	snapshot.addAll(this.pendingSplits);
	return snapshot;
}
Example #4
Source File: ContinuousFileReaderOperator.java    From flink with Apache License 2.0 5 votes vote down vote up 
private List<T> getReaderState() throws IOException {
	List<T> snapshot = new ArrayList<>(splits.size());
	if (currentSplit != null) {
		if (this.format instanceof CheckpointableInputFormat && state == ReaderState.READING) {
			Serializable formatState =
					((CheckpointableInputFormat<T, Serializable>) this.format).getCurrentState();
			this.currentSplit.setSplitState(formatState);
		}
		snapshot.add(this.currentSplit);
	}
	snapshot.addAll(splits);
	return snapshot;
}