org.apache.flink.runtime.io.network.api.CheckpointBarrier Java Examples

@Test
public void test() throws Exception {
	TestInvokable invokable = new TestInvokable();
	CheckpointBarrierUnaligner unaligner = new CheckpointBarrierUnaligner(TestSubtaskCheckpointCoordinator.INSTANCE, "test", invokable, new MockIndexedInputGate(0, numChannels));

	for (RuntimeEvent e : events) {
		if (e instanceof CancelCheckpointMarker) {
			unaligner.processCancellationBarrier((CancelCheckpointMarker) e);
		} else if (e instanceof CheckpointBarrier) {
			unaligner.processBarrier((CheckpointBarrier) e, new InputChannelInfo(0, channel));
		} else {
			throw new IllegalArgumentException("unexpected event type: " + e);
		}
	}

	assertEquals("expectAbortCheckpoint", expectAbortCheckpoint, invokable.checkpointAborted);
	assertEquals("expectTriggerCheckpoint", expectTriggerCheckpoint, invokable.checkpointTriggered);
}
 

private synchronized void handleNewCheckpoint(CheckpointBarrier barrier) throws IOException {
	long barrierId = barrier.getId();
	if (!allBarriersReceivedFuture.isDone()) {
		CheckpointException exception = new CheckpointException("Barrier id: " + barrierId, CHECKPOINT_DECLINED_SUBSUMED);
		if (isCheckpointPending()) {
			// we did not complete the current checkpoint, another started before
			LOG.warn("{}: Received checkpoint barrier for checkpoint {} before completing current checkpoint {}. " +
					"Skipping current checkpoint.",
				handler.taskName,
				barrierId,
				currentReceivedCheckpointId);

			// let the task know we are not completing this
			final long currentCheckpointId = currentReceivedCheckpointId;
			handler.executeInTaskThread(() -> handler.notifyAbort(currentCheckpointId, exception), "notifyAbort");
		}
		allBarriersReceivedFuture.completeExceptionally(exception);
	}

	currentReceivedCheckpointId = barrierId;
	storeNewBuffers.entrySet().forEach(storeNewBuffer -> storeNewBuffer.setValue(true));
	numBarriersReceived = 0;
	allBarriersReceivedFuture = new CompletableFuture<>();
	checkpointCoordinator.initCheckpoint(barrierId, barrier.getCheckpointOptions());
}
 

/**
 * Validates that the buffer preserved the order of elements for a
 * input with a single input channel, and checkpoint events.
 */
@Test
public void testSingleChannelWithBarriers() throws Exception {
	BufferOrEvent[] sequence = {
		createBuffer(0), createBuffer(0), createBuffer(0),
		createBarrier(1, 0),
		createBuffer(0), createBuffer(0), createBuffer(0), createBuffer(0),
		createBarrier(2, 0), createBarrier(3, 0),
		createBuffer(0), createBuffer(0),
		createBarrier(4, 0), createBarrier(5, 0), createBarrier(6, 0),
		createBuffer(0), createEndOfPartition(0)
	};
	ValidatingCheckpointHandler handler = new ValidatingCheckpointHandler();
	inputGate = createBarrierBuffer(1, sequence, handler);

	handler.setNextExpectedCheckpointId(1L);

	for (BufferOrEvent boe : sequence) {
		if (boe.isBuffer() || boe.getEvent().getClass() != CheckpointBarrier.class) {
			assertEquals(boe, inputGate.pollNext().get());
		}
	}
}
 

@Override
public Optional<BufferOrEvent> getNextBufferOrEvent() throws IOException, InterruptedException {
	currentChannel = (currentChannel + 1) % numberOfChannels;

	if (barrierGens[currentChannel].isNextBarrier()) {
		return Optional.of(
			new BufferOrEvent(
				new CheckpointBarrier(
					++currentBarriers[currentChannel],
					System.currentTimeMillis(),
					CheckpointOptions.forCheckpointWithDefaultLocation()),
				currentChannel));
	} else {
		Buffer buffer = bufferPools[currentChannel].requestBuffer();
		buffer.getMemorySegment().putLong(0, c++);
		return Optional.of(new BufferOrEvent(buffer, currentChannel));
	}
}
 

@Test
public void testMissingCancellationBarriers() throws Exception {
	BufferOrEvent[] sequence = {
		createBarrier(1L, 0),
		createCancellationBarrier(2L, 0),
		createCancellationBarrier(3L, 0),
		createCancellationBarrier(3L, 1),
		createBuffer(0)
	};
	AbstractInvokable validator = new CheckpointSequenceValidator(-3);
	inputGate = createBarrierBuffer(2, sequence, validator);

	for (BufferOrEvent boe : sequence) {
		if (boe.isBuffer() || (boe.getEvent().getClass() != CheckpointBarrier.class && boe.getEvent().getClass() != CancelCheckpointMarker.class)) {
			assertEquals(boe, inputGate.pollNext().get());
		}
	}
}
 

@Override
public void processBarrier(CheckpointBarrier receivedBarrier, InputChannelInfo channelInfo) throws Exception {
	if (receivedBarrier.getId() < lastSeenBarrierId) {
		return;
	}

	lastSeenBarrierId = receivedBarrier.getId();
	CheckpointBarrierHandler previousHandler = activeHandler;
	activeHandler = receivedBarrier.isCheckpoint() ? unalignedHandler : alignedHandler;
	if (previousHandler != activeHandler) {
		previousHandler.abortPendingCheckpoint(
			lastSeenBarrierId,
			new CheckpointException(format("checkpoint subsumed by %d", lastSeenBarrierId), CHECKPOINT_DECLINED_SUBSUMED));
	}

	activeHandler.processBarrier(receivedBarrier, channelInfo);
}
 

@Override
public Optional<BufferOrEvent> getNext() throws IOException, InterruptedException {
	currentChannel = (currentChannel + 1) % numberOfChannels;

	if (barrierGens[currentChannel].isNextBarrier()) {
		return Optional.of(
			new BufferOrEvent(
				new CheckpointBarrier(
					++currentBarriers[currentChannel],
					System.currentTimeMillis(),
					CheckpointOptions.forCheckpointWithDefaultLocation()),
				currentChannel));
	} else {
		Buffer buffer = bufferPools[currentChannel].requestBuffer();
		buffer.getMemorySegment().putLong(0, c++);
		return Optional.of(new BufferOrEvent(buffer, currentChannel));
	}
}
 

@Test
public void testPreviousHandlerReset() throws Exception {
	SingleInputGate inputGate = new SingleInputGateBuilder().setNumberOfChannels(2).build();
	inputGate.setInputChannels(new TestInputChannel(inputGate, 0), new TestInputChannel(inputGate, 1));
	TestInvokable target = new TestInvokable();
	CheckpointBarrierAligner alignedHandler = new CheckpointBarrierAligner("test", target, inputGate);
	CheckpointBarrierUnaligner unalignedHandler = new CheckpointBarrierUnaligner(TestSubtaskCheckpointCoordinator.INSTANCE, "test", target, inputGate);
	AlternatingCheckpointBarrierHandler barrierHandler = new AlternatingCheckpointBarrierHandler(alignedHandler, unalignedHandler, target);

	for (int i = 0; i < 4; i++) {
		int channel = i % 2;
		CheckpointType type = channel == 0 ? CHECKPOINT : SAVEPOINT;
		barrierHandler.processBarrier(new CheckpointBarrier(i, 0, new CheckpointOptions(type, CheckpointStorageLocationReference.getDefault())), new InputChannelInfo(0, channel));
		assertEquals(type.isSavepoint(), alignedHandler.isCheckpointPending());
		assertNotEquals(alignedHandler.isCheckpointPending(), unalignedHandler.isCheckpointPending());

		if (!type.isSavepoint()) {
			assertFalse(barrierHandler.getAllBarriersReceivedFuture(i).isDone());
			assertInflightDataEquals(unalignedHandler, barrierHandler, i, inputGate.getNumberOfInputChannels());
		}
	}
}
 

/**
 * We still need to trigger checkpoint via {@link ThreadSafeUnaligner#notifyBarrierReceived(CheckpointBarrier, InputChannelInfo)}
 * while reading the first barrier from one channel, because this might happen
 * earlier than the previous async trigger via mailbox by netty thread.
 *
 * <p>Note this is also suitable for the trigger case of local input channel.
 */
@Override
public void processBarrier(CheckpointBarrier receivedBarrier, InputChannelInfo channelInfo) throws Exception {
	long barrierId = receivedBarrier.getId();
	if (currentConsumedCheckpointId > barrierId || (currentConsumedCheckpointId == barrierId && !isCheckpointPending())) {
		// ignore old and cancelled barriers
		return;
	}
	if (currentConsumedCheckpointId < barrierId) {
		currentConsumedCheckpointId = barrierId;
		numBarrierConsumed = 0;
		hasInflightBuffers.entrySet().forEach(hasInflightBuffer -> hasInflightBuffer.setValue(true));
	}
	if (currentConsumedCheckpointId == barrierId) {
		hasInflightBuffers.put(channelInfo, false);
		numBarrierConsumed++;
	}
	threadSafeUnaligner.notifyBarrierReceived(receivedBarrier, channelInfo);
}
 

private void notifyCheckpoint(CheckpointBarrier checkpointBarrier) throws Exception {
	if (toNotifyOnCheckpoint != null) {
		CheckpointMetaData checkpointMetaData =
				new CheckpointMetaData(checkpointBarrier.getId(), checkpointBarrier.getTimestamp());

		long bytesBuffered = currentBuffered != null ? currentBuffered.size() : 0L;

		CheckpointMetrics checkpointMetrics = new CheckpointMetrics()
				.setBytesBufferedInAlignment(bytesBuffered)
				.setAlignmentDurationNanos(latestAlignmentDurationNanos);

		toNotifyOnCheckpoint.triggerCheckpointOnBarrier(
			checkpointMetaData,
			checkpointBarrier.getCheckpointOptions(),
			checkpointMetrics);
	}
}
 

@Override
public boolean notifyPriorityEvent(BufferConsumer eventBufferConsumer) throws IOException {
	if (inputGate.getBufferReceivedListener() == null) {
		// in rare cases and very low checkpointing intervals, we may receive the first barrier, before setting
		// up CheckpointedInputGate
		return false;
	}
	Buffer buffer = eventBufferConsumer.build();
	try {
		CheckpointBarrier event = parseCheckpointBarrierOrNull(buffer);
		if (event == null) {
			throw new IllegalStateException("Currently only checkpoint barriers are known priority events");
		} else if (event.isCheckpoint()) {
			inputGate.getBufferReceivedListener().notifyBarrierReceived(event, channelInfo);
		}
	} finally {
		buffer.recycleBuffer();
	}
	// already processed
	return true;
}
 

@Test
public void testSerializeDeserializeEvent() throws Exception {
	AbstractEvent[] events = {
			EndOfPartitionEvent.INSTANCE,
			EndOfSuperstepEvent.INSTANCE,
			new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forCheckpointWithDefaultLocation()),
			new TestTaskEvent(Math.random(), 12361231273L),
			new CancelCheckpointMarker(287087987329842L)
	};

	for (AbstractEvent evt : events) {
		ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
		assertTrue(serializedEvent.hasRemaining());

		AbstractEvent deserialized =
				EventSerializer.fromSerializedEvent(serializedEvent, getClass().getClassLoader());
		assertNotNull(deserialized);
		assertEquals(evt, deserialized);
	}
}
 

@Override
public void spillInflightBuffers(long checkpointId, ChannelStateWriter channelStateWriter) throws IOException {
	synchronized (receivedBuffers) {
		checkState(checkpointId > lastRequestedCheckpointId, "Need to request the next checkpointId");

		final List<Buffer> inflightBuffers = new ArrayList<>(receivedBuffers.size());
		for (Buffer buffer : receivedBuffers) {
			CheckpointBarrier checkpointBarrier = parseCheckpointBarrierOrNull(buffer);
			if (checkpointBarrier != null && checkpointBarrier.getId() >= checkpointId) {
				break;
			}
			if (buffer.isBuffer()) {
				inflightBuffers.add(buffer.retainBuffer());
			}
		}

		lastRequestedCheckpointId = checkpointId;

		channelStateWriter.addInputData(
			checkpointId,
			channelInfo,
			ChannelStateWriter.SEQUENCE_NUMBER_UNKNOWN,
			CloseableIterator.fromList(inflightBuffers, Buffer::recycleBuffer));
	}
}
 

@Test
public void testToBufferConsumer() throws IOException {
	for (AbstractEvent evt : events) {
		BufferConsumer bufferConsumer = EventSerializer.toBufferConsumer(evt);

		assertFalse(bufferConsumer.isBuffer());
		assertTrue(bufferConsumer.isFinished());
		assertTrue(bufferConsumer.isDataAvailable());
		assertFalse(bufferConsumer.isRecycled());

		if (evt instanceof CheckpointBarrier) {
			assertTrue(bufferConsumer.build().getDataType().isBlockingUpstream());
		} else {
			assertEquals(Buffer.DataType.EVENT_BUFFER, bufferConsumer.build().getDataType());
		}
	}
}
 

@Test
public void testToBuffer() throws IOException {
	for (AbstractEvent evt : events) {
		Buffer buffer = EventSerializer.toBuffer(evt);

		assertFalse(buffer.isBuffer());
		assertTrue(buffer.readableBytes() > 0);
		assertFalse(buffer.isRecycled());

		if (evt instanceof CheckpointBarrier) {
			assertTrue(buffer.getDataType().isBlockingUpstream());
		} else {
			assertEquals(Buffer.DataType.EVENT_BUFFER, buffer.getDataType());
		}
	}
}
 

@Test
public void testSerializeDeserializeEvent() throws Exception {
	AbstractEvent[] events = {
			EndOfPartitionEvent.INSTANCE,
			EndOfSuperstepEvent.INSTANCE,
			new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forCheckpointWithDefaultLocation()),
			new TestTaskEvent(Math.random(), 12361231273L),
			new CancelCheckpointMarker(287087987329842L)
	};

	for (AbstractEvent evt : events) {
		ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
		assertTrue(serializedEvent.hasRemaining());

		AbstractEvent deserialized =
				EventSerializer.fromSerializedEvent(serializedEvent, getClass().getClassLoader());
		assertNotNull(deserialized);
		assertEquals(evt, deserialized);
	}
}
 

@Test
public void testMultiChannelWithBarriers() throws Exception {
	BufferOrEvent[] sequence = {
			createBuffer(0), createBuffer(2), createBuffer(0),
			createBarrier(1, 1), createBarrier(1, 2),
			createBuffer(2), createBuffer(1),
			createBarrier(1, 0),

			createBuffer(0), createBuffer(0), createBuffer(1), createBuffer(1), createBuffer(2),
			createBarrier(2, 0), createBarrier(2, 1), createBarrier(2, 2),

			createBuffer(2), createBuffer(2),
			createBarrier(3, 2),
			createBuffer(2), createBuffer(2),
			createBarrier(3, 0), createBarrier(3, 1),

			createBarrier(4, 1), createBarrier(4, 2), createBarrier(4, 0),

			createBuffer(0)
	};
	CheckpointSequenceValidator validator =
		new CheckpointSequenceValidator(1, 2, 3, 4);
	inputGate = createBarrierTracker(3, sequence, validator);

	for (BufferOrEvent boe : sequence) {
		if (boe.isBuffer() || boe.getEvent().getClass() != CheckpointBarrier.class) {
			assertEquals(boe, inputGate.pollNext().get());
		}
	}
}
 

@Test
public void testMultiChannelSkippingCheckpoints() throws Exception {
	BufferOrEvent[] sequence = {
			createBuffer(0), createBuffer(2), createBuffer(0),
			createBarrier(1, 1), createBarrier(1, 2),
			createBuffer(2), createBuffer(1),
			createBarrier(1, 0),

			createBuffer(0), createBuffer(0), createBuffer(1), createBuffer(1), createBuffer(2),
			createBarrier(2, 0), createBarrier(2, 1), createBarrier(2, 2),

			createBuffer(2), createBuffer(2),
			createBarrier(3, 2),
			createBuffer(2), createBuffer(2),

			// jump to checkpoint 4
			createBarrier(4, 0),
			createBuffer(0), createBuffer(1), createBuffer(2),
			createBarrier(4, 1),
			createBuffer(1),
			createBarrier(4, 2),

			createBuffer(0)
	};
	CheckpointSequenceValidator validator =
		new CheckpointSequenceValidator(1, 2, 4);
	inputGate = createBarrierTracker(3, sequence, validator);

	for (BufferOrEvent boe : sequence) {
		if (boe.isBuffer() || boe.getEvent().getClass() != CheckpointBarrier.class) {
			assertEquals(boe, inputGate.pollNext().get());
		}
	}
}
 

private static ByteBuffer serializeCheckpointBarrier(CheckpointBarrier barrier) throws IOException {
	final CheckpointOptions checkpointOptions = barrier.getCheckpointOptions();
	final CheckpointType checkpointType = checkpointOptions.getCheckpointType();

	final byte[] locationBytes = checkpointOptions.getTargetLocation().isDefaultReference() ?
			null : checkpointOptions.getTargetLocation().getReferenceBytes();

	final ByteBuffer buf = ByteBuffer.allocate(30 + (locationBytes == null ? 0 : locationBytes.length));

	// we do not use checkpointType.ordinal() here to make the serialization robust
	// against changes in the enum (such as changes in the order of the values)
	final int typeInt;
	if (checkpointType == CheckpointType.CHECKPOINT) {
		typeInt = CHECKPOINT_TYPE_CHECKPOINT;
	} else if (checkpointType == CheckpointType.SAVEPOINT) {
		typeInt = CHECKPOINT_TYPE_SAVEPOINT;
	} else if (checkpointType == CheckpointType.SYNC_SAVEPOINT) {
		typeInt = CHECKPOINT_TYPE_SYNC_SAVEPOINT;
	} else {
		throw new IOException("Unknown checkpoint type: " + checkpointType);
	}

	buf.putInt(CHECKPOINT_BARRIER_EVENT);
	buf.putLong(barrier.getId());
	buf.putLong(barrier.getTimestamp());
	buf.putInt(typeInt);

	if (locationBytes == null) {
		buf.putInt(-1);
	} else {
		buf.putInt(locationBytes.length);
		buf.put(locationBytes);
	}
	buf.put((byte) (checkpointOptions.isExactlyOnceMode() ? 1 : 0));
	buf.put((byte) (checkpointOptions.isUnalignedCheckpoint() ? 1 : 0));

	buf.flip();
	return buf;
}
 

private void testCheckpointBarrierSerialization(CheckpointOptions options) throws IOException {
	final long checkpointId = Integer.MAX_VALUE + 123123L;
	final long timestamp = Integer.MAX_VALUE + 1228L;

	final CheckpointBarrier barrierBeforeSerialization = new CheckpointBarrier(checkpointId, timestamp, options);
	final CheckpointBarrier barrierAfterDeserialization = serializeAndDeserializeCheckpointBarrier(barrierBeforeSerialization);

	assertEquals(barrierBeforeSerialization, barrierAfterDeserialization);
}
 

@Test
public void testBarrierOvertaking() throws Exception {
	subpartition.add(createFilledFinishedBufferConsumer(1));
	assertEquals(0, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	subpartition.add(createFilledFinishedBufferConsumer(2));
	assertEquals(1, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	BufferConsumer eventBuffer = EventSerializer.toBufferConsumer(EndOfSuperstepEvent.INSTANCE);
	subpartition.add(eventBuffer);
	assertEquals(1, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	subpartition.add(createFilledFinishedBufferConsumer(4));
	assertEquals(1, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	CheckpointOptions options = new CheckpointOptions(
		CheckpointType.CHECKPOINT,
		new CheckpointStorageLocationReference(new byte[]{0, 1, 2}),
		true,
		true);
	BufferConsumer barrierBuffer = EventSerializer.toBufferConsumer(new CheckpointBarrier(0, 0, options));
	subpartition.add(barrierBuffer, true);
	assertEquals(2, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	List<Buffer> inflight = subpartition.requestInflightBufferSnapshot();
	assertEquals(Arrays.asList(1, 2, 4), inflight.stream().map(Buffer::getSize).collect(Collectors.toList()));
	inflight.forEach(Buffer::recycleBuffer);

	assertNextEvent(readView, barrierBuffer.getWrittenBytes(), CheckpointBarrier.class, true, 2, false, true);
	assertNextBuffer(readView, 1, true, 1, false, true);
	assertNextBuffer(readView, 2, true, 0, true, true);
	assertNextEvent(readView, eventBuffer.getWrittenBytes(), EndOfSuperstepEvent.class, false, 0, false, true);
	assertNextBuffer(readView, 4, false, 0, false, true);
	assertNoNextBuffer(readView);
}
 

@Test
public void testNoNotifyOnSavepoint() throws IOException {
	TestBufferReceivedListener listener = new TestBufferReceivedListener();
	SingleInputGate inputGate = new SingleInputGateBuilder().build();
	inputGate.registerBufferReceivedListener(listener);
	RemoteInputChannel channel = InputChannelBuilder.newBuilder().buildRemoteChannel(inputGate);
	channel.onBuffer(toBuffer(new CheckpointBarrier(123L, 123L, new CheckpointOptions(SAVEPOINT, CheckpointStorageLocationReference.getDefault()))), 0, 0);
	channel.checkError();
	assertTrue(listener.notifiedOnBarriers.isEmpty());
}
 

public static ByteBuffer toSerializedEvent(AbstractEvent event) throws IOException {
	final Class<?> eventClass = event.getClass();
	if (eventClass == EndOfPartitionEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_PARTITION_EVENT });
	}
	else if (eventClass == CheckpointBarrier.class) {
		return serializeCheckpointBarrier((CheckpointBarrier) event);
	}
	else if (eventClass == EndOfSuperstepEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_SUPERSTEP_EVENT });
	}
	else if (eventClass == EndOfChannelStateEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_CHANNEL_STATE_EVENT });
	}
	else if (eventClass == CancelCheckpointMarker.class) {
		CancelCheckpointMarker marker = (CancelCheckpointMarker) event;

		ByteBuffer buf = ByteBuffer.allocate(12);
		buf.putInt(0, CANCEL_CHECKPOINT_MARKER_EVENT);
		buf.putLong(4, marker.getCheckpointId());
		return buf;
	}
	else {
		try {
			final DataOutputSerializer serializer = new DataOutputSerializer(128);
			serializer.writeInt(OTHER_EVENT);
			serializer.writeUTF(event.getClass().getName());
			event.write(serializer);
			return serializer.wrapAsByteBuffer();
		}
		catch (IOException e) {
			throw new IOException("Error while serializing event.", e);
		}
	}
}
 

private static ByteBuffer serializeCheckpointBarrier(CheckpointBarrier barrier) throws IOException {
	final CheckpointOptions checkpointOptions = barrier.getCheckpointOptions();
	final CheckpointType checkpointType = checkpointOptions.getCheckpointType();

	final byte[] locationBytes = checkpointOptions.getTargetLocation().isDefaultReference() ?
			null : checkpointOptions.getTargetLocation().getReferenceBytes();

	final ByteBuffer buf = ByteBuffer.allocate(28 + (locationBytes == null ? 0 : locationBytes.length));

	// we do not use checkpointType.ordinal() here to make the serialization robust
	// against changes in the enum (such as changes in the order of the values)
	final int typeInt;
	if (checkpointType == CheckpointType.CHECKPOINT) {
		typeInt = CHECKPOINT_TYPE_CHECKPOINT;
	} else if (checkpointType == CheckpointType.SAVEPOINT) {
		typeInt = CHECKPOINT_TYPE_SAVEPOINT;
	} else if (checkpointType == CheckpointType.SYNC_SAVEPOINT) {
		typeInt = CHECKPOINT_TYPE_SYNC_SAVEPOINT;
	} else {
		throw new IOException("Unknown checkpoint type: " + checkpointType);
	}

	buf.putInt(CHECKPOINT_BARRIER_EVENT);
	buf.putLong(barrier.getId());
	buf.putLong(barrier.getTimestamp());
	buf.putInt(typeInt);

	if (locationBytes == null) {
		buf.putInt(-1);
	} else {
		buf.putInt(locationBytes.length);
		buf.put(locationBytes);
	}

	buf.flip();
	return buf;
}
 

@Override
public InputStatus emitNext(DataOutput<T> output) throws Exception {

	while (true) {
		// get the stream element from the deserializer
		if (currentRecordDeserializer != null) {
			DeserializationResult result = currentRecordDeserializer.getNextRecord(deserializationDelegate);
			if (result.isBufferConsumed()) {
				currentRecordDeserializer.getCurrentBuffer().recycleBuffer();
				currentRecordDeserializer = null;
			}

			if (result.isFullRecord()) {
				processElement(deserializationDelegate.getInstance(), output);
				return InputStatus.MORE_AVAILABLE;
			}
		}

		Optional<BufferOrEvent> bufferOrEvent = checkpointedInputGate.pollNext();
		if (bufferOrEvent.isPresent()) {
			// return to the mailbox after receiving a checkpoint barrier to avoid processing of
			// data after the barrier before checkpoint is performed for unaligned checkpoint mode
			if (bufferOrEvent.get().isEvent() && bufferOrEvent.get().getEvent() instanceof CheckpointBarrier) {
				return InputStatus.MORE_AVAILABLE;
			}
			processBufferOrEvent(bufferOrEvent.get());
		} else {
			if (checkpointedInputGate.isFinished()) {
				checkState(checkpointedInputGate.getAvailableFuture().isDone(), "Finished BarrierHandler should be available");
				return InputStatus.END_OF_INPUT;
			}
			return InputStatus.NOTHING_AVAILABLE;
		}
	}
}
 

private void testCheckpointBarrierSerialization(CheckpointOptions options) throws IOException {
	final long checkpointId = Integer.MAX_VALUE + 123123L;
	final long timestamp = Integer.MAX_VALUE + 1228L;

	final CheckpointBarrier barrierBeforeSerialization = new CheckpointBarrier(checkpointId, timestamp, options);
	final CheckpointBarrier barrierAfterDeserialization = serializeAndDeserializeCheckpointBarrier(barrierBeforeSerialization);

	assertEquals(barrierBeforeSerialization, barrierAfterDeserialization);
}
 

private BufferOrEvent createBarrier(long checkpointId, int channel) {
	sizeCounter++;
	return new BufferOrEvent(
		new CheckpointBarrier(
			checkpointId,
			System.currentTimeMillis(),
			CheckpointOptions.forCheckpointWithDefaultLocation()),
		new InputChannelInfo(0, channel));
}
 

/**
 * InputGate on CheckpointBarrier can enqueue a mailbox action to execute and StreamTaskNetworkInput must
 * allow this action to execute before processing a following record.
 */
@Test
public void testNoDataProcessedAfterCheckpointBarrier() throws Exception {
	CheckpointBarrier barrier = new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation());

	List<BufferOrEvent> buffers = new ArrayList<>(2);
	buffers.add(new BufferOrEvent(barrier, new InputChannelInfo(0, 0)));
	buffers.add(createDataBuffer());

	VerifyRecordsDataOutput output = new VerifyRecordsDataOutput<>();
	StreamTaskNetworkInput input = createStreamTaskNetworkInput(buffers, output);

	assertHasNextElement(input, output);
	assertEquals(0, output.getNumberOfEmittedRecords());
}
 

@Test
public void testHasInflightDataBeforeProcessBarrier() throws Exception {
	SingleInputGate inputGate = new SingleInputGateBuilder().setNumberOfChannels(2).build();
	inputGate.setInputChannels(new TestInputChannel(inputGate, 0), new TestInputChannel(inputGate, 1));
	TestInvokable target = new TestInvokable();
	CheckpointBarrierAligner alignedHandler = new CheckpointBarrierAligner("test", target, inputGate);
	CheckpointBarrierUnaligner unalignedHandler = new CheckpointBarrierUnaligner(TestSubtaskCheckpointCoordinator.INSTANCE, "test", target, inputGate);
	AlternatingCheckpointBarrierHandler barrierHandler = new AlternatingCheckpointBarrierHandler(alignedHandler, unalignedHandler, target);

	final long id = 1;
	unalignedHandler.processBarrier(new CheckpointBarrier(id, 0, new CheckpointOptions(CHECKPOINT, CheckpointStorageLocationReference.getDefault())), new InputChannelInfo(0, 0));

	assertInflightDataEquals(unalignedHandler, barrierHandler, id, inputGate.getNumberOfInputChannels());
	assertFalse(barrierHandler.getAllBarriersReceivedFuture(id).isDone());
}
 

/**
 * Tests broadcasting events when no records have been emitted yet.
 */
@Test
public void testBroadcastEventNoRecords() throws Exception {
	int numberOfChannels = 4;
	int bufferSize = 32;

	@SuppressWarnings("unchecked")
	Queue<BufferConsumer>[] queues = new Queue[numberOfChannels];
	for (int i = 0; i < numberOfChannels; i++) {
		queues[i] = new ArrayDeque<>();
	}

	TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize);

	ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider);
	RecordWriter<ByteArrayIO> writer = new RecordWriterBuilder().build(partitionWriter);
	CheckpointBarrier barrier = new CheckpointBarrier(Integer.MAX_VALUE + 919192L, Integer.MAX_VALUE + 18828228L, CheckpointOptions.forCheckpointWithDefaultLocation());

	// No records emitted yet, broadcast should not request a buffer
	writer.broadcastEvent(barrier);

	assertEquals(0, bufferProvider.getNumberOfCreatedBuffers());

	for (int i = 0; i < numberOfChannels; i++) {
		assertEquals(1, queues[i].size());
		BufferOrEvent boe = parseBuffer(queues[i].remove(), i);
		assertTrue(boe.isEvent());
		assertEquals(barrier, boe.getEvent());
		assertEquals(0, queues[i].size());
	}
}