org.apache.flink.streaming.runtime.streamrecord.StreamElement Java Examples

@SuppressWarnings("unchecked")
public RecordWriterOutput(
		RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter,
		TypeSerializer<OUT> outSerializer,
		OutputTag outputTag,
		StreamStatusProvider streamStatusProvider) {

	checkNotNull(recordWriter);
	this.outputTag = outputTag;
	// generic hack: cast the writer to generic Object type so we can use it
	// with multiplexed records and watermarks
	this.recordWriter = (RecordWriter<SerializationDelegate<StreamElement>>)
			(RecordWriter<?>) recordWriter;

	TypeSerializer<StreamElement> outRecordSerializer =
			new StreamElementSerializer<>(outSerializer);

	if (outSerializer != null) {
		serializationDelegate = new SerializationDelegate<StreamElement>(outRecordSerializer);
	}

	this.streamStatusProvider = checkNotNull(streamStatusProvider);
}
 

@Test
public void testNoMaxWatermarkOnImmediateCancel() throws Exception {

	final List<StreamElement> output = new ArrayList<>();

	// regular stream source operator
	final StreamSource<String, InfiniteSource<String>> operator =
			new StreamSource<>(new InfiniteSource<String>());

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0);
	operator.cancel();

	// run and exit
	operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output));

	assertTrue(output.isEmpty());
}
 

@SuppressWarnings("unchecked")
public RecordWriterOutput(
		RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter,
		TypeSerializer<OUT> outSerializer,
		OutputTag outputTag,
		StreamStatusProvider streamStatusProvider) {

	checkNotNull(recordWriter);
	this.outputTag = outputTag;
	// generic hack: cast the writer to generic Object type so we can use it
	// with multiplexed records and watermarks
	this.recordWriter = (RecordWriter<SerializationDelegate<StreamElement>>)
			(RecordWriter<?>) recordWriter;

	TypeSerializer<StreamElement> outRecordSerializer =
			new StreamElementSerializer<>(outSerializer);

	if (outSerializer != null) {
		serializationDelegate = new SerializationDelegate<StreamElement>(outRecordSerializer);
	}

	this.streamStatusProvider = checkNotNull(streamStatusProvider);
}
 

@Test
public void testNoMaxWatermarkOnImmediateStop() throws Exception {

	final List<StreamElement> output = new ArrayList<>();

	// regular stream source operator
	final StoppableStreamSource<String, InfiniteSource<String>> operator =
			new StoppableStreamSource<>(new InfiniteSource<String>());

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0);
	operator.stop();

	// run and stop
	operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output));

	assertTrue(output.isEmpty());
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	ListState<StreamElement> partitionableState =
		getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer));
	partitionableState.clear();

	Collection<StreamElementQueueEntry<?>> values = queue.values();

	try {
		for (StreamElementQueueEntry<?> value : values) {
			partitionableState.add(value.getStreamElement());
		}

		// add the pending stream element queue entry if the stream element queue is currently full
		if (pendingStreamElementQueueEntry != null) {
			partitionableState.add(pendingStreamElementQueueEntry.getStreamElement());
		}
	} catch (Exception e) {
		partitionableState.clear();

		throw new Exception("Could not add stream element queue entries to operator state " +
			"backend of operator " + getOperatorName() + '.', e);
	}
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	ListState<StreamElement> partitionableState =
		getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer));
	partitionableState.clear();

	Collection<StreamElementQueueEntry<?>> values = queue.values();

	try {
		for (StreamElementQueueEntry<?> value : values) {
			partitionableState.add(value.getStreamElement());
		}

		// add the pending stream element queue entry if the stream element queue is currently full
		if (pendingStreamElementQueueEntry != null) {
			partitionableState.add(pendingStreamElementQueueEntry.getStreamElement());
		}
	} catch (Exception e) {
		partitionableState.clear();

		throw new Exception("Could not add stream element queue entries to operator state " +
			"backend of operator " + getOperatorName() + '.', e);
	}
}
 

private void processElement(StreamElement recordOrMark, int channel) throws Exception {
	if (recordOrMark.isRecord()) {
		// now we can do the actual processing
		StreamRecord<IN> record = recordOrMark.asRecord();
		synchronized (lock) {
			numRecordsIn.inc();
			streamOperator.setKeyContextElement1(record);
			streamOperator.processElement(record);
		}
	}
	else if (recordOrMark.isWatermark()) {
		// handle watermark
		statusWatermarkValve.inputWatermark(recordOrMark.asWatermark(), channel);
	} else if (recordOrMark.isStreamStatus()) {
		// handle stream status
		statusWatermarkValve.inputStreamStatus(recordOrMark.asStreamStatus(), channel);
	} else if (recordOrMark.isLatencyMarker()) {
		// handle latency marker
		synchronized (lock) {
			streamOperator.processLatencyMarker(recordOrMark.asLatencyMarker());
		}
	} else {
		throw new UnsupportedOperationException("Unknown type of StreamElement");
	}
}
 

private void processElement1(StreamElement recordOrMark, int channel) throws Exception {
	if (recordOrMark.isRecord()) {
		StreamRecord<IN1> record = recordOrMark.asRecord();
		synchronized (lock) {
			numRecordsIn.inc();
			streamOperator.setKeyContextElement1(record);
			streamOperator.processElement1(record);
			inputSelection = inputSelector.nextSelection();
		}
	}
	else if (recordOrMark.isWatermark()) {
		statusWatermarkValve1.inputWatermark(recordOrMark.asWatermark(), channel);
	} else if (recordOrMark.isStreamStatus()) {
		statusWatermarkValve1.inputStreamStatus(recordOrMark.asStreamStatus(), channel);
	} else if (recordOrMark.isLatencyMarker()) {
		synchronized (lock) {
			streamOperator.processLatencyMarker1(recordOrMark.asLatencyMarker());
		}
	} else {
		throw new UnsupportedOperationException("Unknown type of StreamElement on input1");
	}
}
 

private void processElement2(StreamElement recordOrMark, int channel) throws Exception {
	if (recordOrMark.isRecord()) {
		StreamRecord<IN2> record = recordOrMark.asRecord();
		synchronized (lock) {
			numRecordsIn.inc();
			streamOperator.setKeyContextElement2(record);
			streamOperator.processElement2(record);
			inputSelection = inputSelector.nextSelection();
		}
	}
	else if (recordOrMark.isWatermark()) {
		statusWatermarkValve2.inputWatermark(recordOrMark.asWatermark(), channel);
	} else if (recordOrMark.isStreamStatus()) {
		statusWatermarkValve2.inputStreamStatus(recordOrMark.asStreamStatus(), channel);
	} else if (recordOrMark.isLatencyMarker()) {
		synchronized (lock) {
			streamOperator.processLatencyMarker2(recordOrMark.asLatencyMarker());
		}
	} else {
		throw new UnsupportedOperationException("Unknown type of StreamElement on input2");
	}
}
 

@VisibleForTesting
StreamTaskNetworkInput(
	CheckpointedInputGate checkpointedInputGate,
	TypeSerializer<?> inputSerializer,
	StatusWatermarkValve statusWatermarkValve,
	int inputIndex,
	RecordDeserializer<DeserializationDelegate<StreamElement>>[] recordDeserializers) {

	this.checkpointedInputGate = checkpointedInputGate;
	this.deserializationDelegate = new NonReusingDeserializationDelegate<>(
		new StreamElementSerializer<>(inputSerializer));
	this.recordDeserializers = recordDeserializers;
	this.statusWatermarkValve = statusWatermarkValve;
	this.inputIndex = inputIndex;
	this.channelIndexes = getChannelIndexes(checkpointedInputGate);
}
 

@SuppressWarnings("unchecked")
public RecordWriterOutput(
		RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter,
		TypeSerializer<OUT> outSerializer,
		OutputTag outputTag,
		StreamStatusProvider streamStatusProvider) {

	checkNotNull(recordWriter);
	this.outputTag = outputTag;
	// generic hack: cast the writer to generic Object type so we can use it
	// with multiplexed records and watermarks
	this.recordWriter = (RecordWriter<SerializationDelegate<StreamElement>>)
			(RecordWriter<?>) recordWriter;

	TypeSerializer<StreamElement> outRecordSerializer =
			new StreamElementSerializer<>(outSerializer);

	if (outSerializer != null) {
		serializationDelegate = new SerializationDelegate<StreamElement>(outRecordSerializer);
	}

	this.streamStatusProvider = checkNotNull(streamStatusProvider);
}
 

@Override
public Optional<ResultFuture<OUT>> tryPut(StreamElement streamElement) {
	if (size() < capacity) {
		StreamElementQueueEntry<OUT> queueEntry;
		if (streamElement.isRecord()) {
			queueEntry = addRecord((StreamRecord<?>) streamElement);
		} else if (streamElement.isWatermark()) {
			queueEntry = addWatermark((Watermark) streamElement);
		} else {
			throw new UnsupportedOperationException("Cannot enqueue " + streamElement);
		}

		numberOfEntries++;

		LOG.debug("Put element into unordered stream element queue. New filling degree " +
				"({}/{}).", size(), capacity);

		return Optional.of(queueEntry);
	} else {
		LOG.debug("Failed to put element into unordered stream element queue because it " +
				"was full ({}/{}).", size(), capacity);

		return Optional.empty();
	}
}
 

@Override
public Optional<ResultFuture<OUT>> tryPut(StreamElement streamElement) {
	if (queue.size() < capacity) {
		StreamElementQueueEntry<OUT> queueEntry = createEntry(streamElement);

		queue.add(queueEntry);

		LOG.debug("Put element into ordered stream element queue. New filling degree " +
			"({}/{}).", queue.size(), capacity);

		return Optional.of(queueEntry);
	} else {
		LOG.debug("Failed to put element into ordered stream element queue because it " +
			"was full ({}/{}).", queue.size(), capacity);

		return Optional.empty();
	}
}
 

@Test
public void testEmitMaxWatermarkForFiniteSource() throws Exception {

	// regular stream source operator
	StreamSource<String, FiniteSource<String>> operator =
			new StreamSource<>(new FiniteSource<String>());

	final List<StreamElement> output = new ArrayList<>();

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0);
	OperatorChain<?, ?> operatorChain = createOperatorChain(operator);
	try {
		operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output), operatorChain);
	} finally {
		operatorChain.releaseOutputs();
	}

	assertEquals(1, output.size());
	assertEquals(Watermark.MAX_WATERMARK, output.get(0));
}
 

@Test
public void testNoMaxWatermarkOnImmediateCancel() throws Exception {

	final List<StreamElement> output = new ArrayList<>();

	// regular stream source operator
	final StreamSource<String, InfiniteSource<String>> operator =
			new StreamSource<>(new InfiniteSource<String>());

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0);
	operator.cancel();

	// run and exit
	OperatorChain<?, ?> operatorChain = createOperatorChain(operator);
	try {
		operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output), operatorChain);
	} finally {
		operatorChain.releaseOutputs();
	}

	assertTrue(output.isEmpty());
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	ListState<StreamElement> partitionableState =
		getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer));
	partitionableState.clear();

	try {
		partitionableState.addAll(queue.values());
	} catch (Exception e) {
		partitionableState.clear();

		throw new Exception("Could not add stream element queue entries to operator state " +
			"backend of operator " + getOperatorName() + '.', e);
	}
}
 

@Override
public void open() throws Exception {
	super.open();

	if (recoveredStreamElements != null) {
		for (StreamElement element : recoveredStreamElements.get()) {
			if (element.isRecord()) {
				processElement(element.<IN>asRecord());
			}
			else if (element.isWatermark()) {
				processWatermark(element.asWatermark());
			}
			else if (element.isLatencyMarker()) {
				processLatencyMarker(element.asLatencyMarker());
			}
			else {
				throw new IllegalStateException("Unknown record type " + element.getClass() +
					" encountered while opening the operator.");
			}
		}
		recoveredStreamElements = null;
	}

}
 

@Override
public boolean processInput() throws Exception {
	if (!isPrepared) {
		// the preparations here are not placed in the constructor because all work in it
		// must be executed after all operators are opened.
		prepareForProcessing();
	}

	int readingInputIndex = selectNextReadingInputIndex();
	if (readingInputIndex == -1) {
		return false;
	}
	lastReadInputIndex = readingInputIndex;

	StreamElement recordOrMark;
	if (readingInputIndex == 0) {
		recordOrMark = input1.pollNextNullable();
		if (recordOrMark != null) {
			processElement1(recordOrMark, input1.getLastChannel());
		}
		checkFinished(input1, lastReadInputIndex);
	} else {
		recordOrMark = input2.pollNextNullable();
		if (recordOrMark != null) {
			processElement2(recordOrMark, input2.getLastChannel());
		}
		checkFinished(input2, lastReadInputIndex);
	}

	if (recordOrMark == null) {
		setUnavailableInput(readingInputIndex);
	}

	return recordOrMark != null;
}
 

@Override
public boolean processInput() throws Exception {
	initializeNumRecordsIn();

	StreamElement recordOrMark = input.pollNextNullable();
	if (recordOrMark != null) {
		int channel = input.getLastChannel();
		checkState(channel != StreamTaskInput.UNSPECIFIED);

		processElement(recordOrMark, channel);
	}
	checkFinished();

	return recordOrMark != null;
}
 

/**
 * Pops all completed elements from the head of this queue.
 *
 * @return Completed elements or empty list if none exists.
 */
static List<StreamElement> popCompleted(StreamElementQueue<Integer> queue) {
	final List<StreamElement> completed = new ArrayList<>();
	TimestampedCollector<Integer> collector = new TimestampedCollector<>(new CollectorOutput<>(completed));
	while (queue.hasCompletedElements()) {
		queue.emitCompletedElement(collector);
	}
	collector.close();
	return completed;
}
 

@Override
public List<StreamElement> values() {
	List<StreamElement> list = new ArrayList<>();
	for (Segment s : segments) {
		s.addPendingElements(list);
	}
	return list;
}
 

private StreamElementQueueEntry<OUT> createEntry(StreamElement streamElement) {
	if (streamElement.isRecord()) {
		return new StreamRecordQueueEntry<>((StreamRecord<?>) streamElement);
	}
	if (streamElement.isWatermark()) {
		return new WatermarkQueueEntry<>((Watermark) streamElement);
	}
	throw new UnsupportedOperationException("Cannot enqueue " + streamElement);
}
 

private void serializeRecord(long value, BufferBuilder bufferBuilder) throws IOException {
	RecordSerializer<SerializationDelegate<StreamElement>> serializer = new SpanningRecordSerializer<>();
	SerializationDelegate<StreamElement> serializationDelegate =
		new SerializationDelegate<>(
			new StreamElementSerializer<>(LongSerializer.INSTANCE));
	serializationDelegate.setInstance(new StreamRecord<>(value));
	serializer.serializeRecord(serializationDelegate);

	assertFalse(serializer.copyToBufferBuilder(bufferBuilder).isFullBuffer());
}
 

@Test
public void testAutomaticWatermarkContext() throws Exception {

	// regular stream source operator
	final StreamSource<String, InfiniteSource<String>> operator =
		new StreamSource<>(new InfiniteSource<>());

	long watermarkInterval = 10;
	TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
	processingTimeService.setCurrentTime(0);

	setupSourceOperator(operator, TimeCharacteristic.IngestionTime, watermarkInterval, processingTimeService);

	final List<StreamElement> output = new ArrayList<>();

	StreamSourceContexts.getSourceContext(TimeCharacteristic.IngestionTime,
		operator.getContainingTask().getProcessingTimeService(),
		operator.getContainingTask().getCheckpointLock(),
		operator.getContainingTask().getStreamStatusMaintainer(),
		new CollectorOutput<String>(output),
		operator.getExecutionConfig().getAutoWatermarkInterval(),
		-1);

	// periodically emit the watermarks
	// even though we start from 1 the watermark are still
	// going to be aligned with the watermark interval.

	for (long i = 1; i < 100; i += watermarkInterval)  {
		processingTimeService.setCurrentTime(i);
	}

	assertTrue(output.size() == 9);

	long nextWatermark = 0;
	for (StreamElement el : output) {
		nextWatermark += watermarkInterval;
		Watermark wm = (Watermark) el;
		assertTrue(wm.getTimestamp() == nextWatermark);
	}
}
 

private void serializeRecord(long value, BufferBuilder bufferBuilder) throws IOException {
	RecordSerializer<SerializationDelegate<StreamElement>> serializer = new SpanningRecordSerializer<>();
	SerializationDelegate<StreamElement> serializationDelegate =
		new SerializationDelegate<>(
			new StreamElementSerializer<>(LongSerializer.INSTANCE));
	serializationDelegate.setInstance(new StreamRecord<>(value));
	serializer.serializeRecord(serializationDelegate);

	assertFalse(serializer.copyToBufferBuilder(bufferBuilder).isFullBuffer());
}
 

@Override
public List<StreamElement> values() {
	List<StreamElement> list = new ArrayList<>(this.queue.size());
	for (StreamElementQueueEntry e : queue) {
		list.add(e.getInputElement());
	}
	return list;
}
 

@Override
public void open() throws Exception {
	super.open();

	// create the emitter
	this.emitter = new Emitter<>(checkpointingLock, output, queue, this);

	// start the emitter thread
	this.emitterThread = new Thread(emitter, "AsyncIO-Emitter-Thread (" + getOperatorName() + ')');
	emitterThread.setDaemon(true);
	emitterThread.start();

	// process stream elements from state, since the Emit thread will start as soon as all
	// elements from previous state are in the StreamElementQueue, we have to make sure that the
	// order to open all operators in the operator chain proceeds from the tail operator to the
	// head operator.
	if (recoveredStreamElements != null) {
		for (StreamElement element : recoveredStreamElements.get()) {
			if (element.isRecord()) {
				processElement(element.<IN>asRecord());
			}
			else if (element.isWatermark()) {
				processWatermark(element.asWatermark());
			}
			else if (element.isLatencyMarker()) {
				processLatencyMarker(element.asLatencyMarker());
			}
			else {
				throw new IllegalStateException("Unknown record type " + element.getClass() +
					" encountered while opening the operator.");
			}
		}
		recoveredStreamElements = null;
	}

}
 

@Override
protected PriorityQueue<StreamRecord<Tuple2<StreamRoute, IN>>> restoreQueueState(DataInputView dataInputView) throws IOException {
    int sizeOfQueue = dataInputView.readInt();
    PriorityQueue<StreamRecord<Tuple2<StreamRoute, IN>>> priorityQueue = new PriorityQueue<>(sizeOfQueue + 1);
    for (int i = 0; i < sizeOfQueue; i++) {
        String streamId = dataInputView.readUTF();
        StreamElement streamElement = getStreamRecordSerializer(streamId).deserialize(dataInputView);
        priorityQueue.offer(streamElement.<Tuple2<StreamRoute, IN>>asRecord());
    }
    return priorityQueue;
}
 

@Override
protected PriorityQueue<StreamRecord<Tuple2<String, IN>>> restoreQueuerState(DataInputView dataInputView) throws IOException {
    int snapshotSize = dataInputView.readInt();
    int sizeOfQueue = snapshotSize > 0 ? snapshotSize : this.INITIAL_PRIORITY_QUEUE_CAPACITY;
    PriorityQueue<StreamRecord<Tuple2<String, IN>>> priorityQueue = new PriorityQueue<>(sizeOfQueue);
    for (int i = 0; i < snapshotSize; i++) {
        String streamId = dataInputView.readUTF();
        StreamElement streamElement = getStreamRecordSerializer(streamId).deserialize(dataInputView);
        priorityQueue.offer(streamElement.<Tuple2<String, IN>>asRecord());
    }
    return priorityQueue;
}
 

@Test
public void testAutomaticWatermarkContext() throws Exception {

	// regular stream source operator
	final StoppableStreamSource<String, InfiniteSource<String>> operator =
		new StoppableStreamSource<>(new InfiniteSource<String>());

	long watermarkInterval = 10;
	TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
	processingTimeService.setCurrentTime(0);

	setupSourceOperator(operator, TimeCharacteristic.IngestionTime, watermarkInterval, processingTimeService);

	final List<StreamElement> output = new ArrayList<>();

	StreamSourceContexts.getSourceContext(TimeCharacteristic.IngestionTime,
		operator.getContainingTask().getProcessingTimeService(),
		operator.getContainingTask().getCheckpointLock(),
		operator.getContainingTask().getStreamStatusMaintainer(),
		new CollectorOutput<String>(output),
		operator.getExecutionConfig().getAutoWatermarkInterval(),
		-1);

	// periodically emit the watermarks
	// even though we start from 1 the watermark are still
	// going to be aligned with the watermark interval.

	for (long i = 1; i < 100; i += watermarkInterval)  {
		processingTimeService.setCurrentTime(i);
	}

	assertTrue(output.size() == 9);

	long nextWatermark = 0;
	for (StreamElement el : output) {
		nextWatermark += watermarkInterval;
		Watermark wm = (Watermark) el;
		assertTrue(wm.getTimestamp() == nextWatermark);
	}
}