org.apache.flink.runtime.io.network.api.CancelCheckpointMarker Java Examples

@Test
public void test() throws Exception {
	TestInvokable invokable = new TestInvokable();
	CheckpointBarrierUnaligner unaligner = new CheckpointBarrierUnaligner(TestSubtaskCheckpointCoordinator.INSTANCE, "test", invokable, new MockIndexedInputGate(0, numChannels));

	for (RuntimeEvent e : events) {
		if (e instanceof CancelCheckpointMarker) {
			unaligner.processCancellationBarrier((CancelCheckpointMarker) e);
		} else if (e instanceof CheckpointBarrier) {
			unaligner.processBarrier((CheckpointBarrier) e, new InputChannelInfo(0, channel));
		} else {
			throw new IllegalArgumentException("unexpected event type: " + e);
		}
	}

	assertEquals("expectAbortCheckpoint", expectAbortCheckpoint, invokable.checkpointAborted);
	assertEquals("expectTriggerCheckpoint", expectTriggerCheckpoint, invokable.checkpointTriggered);
}
 

/**
 * This test checks that tasks emit a proper cancel checkpoint barrier, if a "trigger checkpoint" message
 * comes before they are ready.
 */
@Test
public void testEmitCancellationBarrierWhenNotReady() throws Exception {
	StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>(
			InitBlockingTask::new, BasicTypeInfo.STRING_TYPE_INFO);
	testHarness.setupOutputForSingletonOperatorChain();

	// start the test - this cannot succeed across the 'init()' method
	testHarness.invoke();

	StreamTask<String, ?> task = testHarness.getTask();

	// tell the task to commence a checkpoint
	boolean result = task.triggerCheckpoint(new CheckpointMetaData(41L, System.currentTimeMillis()),
		CheckpointOptions.forCheckpointWithDefaultLocation(), false);
	assertFalse("task triggered checkpoint though not ready", result);

	// a cancellation barrier should be downstream
	Object emitted = testHarness.getOutput().poll();
	assertNotNull("nothing emitted", emitted);
	assertTrue("wrong type emitted", emitted instanceof CancelCheckpointMarker);
	assertEquals("wrong checkpoint id", 41L, ((CancelCheckpointMarker) emitted).getCheckpointId());
}
 

@Test
public void testSerializeDeserializeEvent() throws Exception {
	AbstractEvent[] events = {
			EndOfPartitionEvent.INSTANCE,
			EndOfSuperstepEvent.INSTANCE,
			new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forCheckpointWithDefaultLocation()),
			new TestTaskEvent(Math.random(), 12361231273L),
			new CancelCheckpointMarker(287087987329842L)
	};

	for (AbstractEvent evt : events) {
		ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
		assertTrue(serializedEvent.hasRemaining());

		AbstractEvent deserialized =
				EventSerializer.fromSerializedEvent(serializedEvent, getClass().getClassLoader());
		assertNotNull(deserialized);
		assertEquals(evt, deserialized);
	}
}
 

@Test
public void testMissingCancellationBarriers() throws Exception {
	BufferOrEvent[] sequence = {
		createBarrier(1L, 0),
		createCancellationBarrier(2L, 0),
		createCancellationBarrier(3L, 0),
		createCancellationBarrier(3L, 1),
		createBuffer(0)
	};
	AbstractInvokable validator = new CheckpointSequenceValidator(-3);
	inputGate = createBarrierBuffer(2, sequence, validator);

	for (BufferOrEvent boe : sequence) {
		if (boe.isBuffer() || (boe.getEvent().getClass() != CheckpointBarrier.class && boe.getEvent().getClass() != CancelCheckpointMarker.class)) {
			assertEquals(boe, inputGate.pollNext().get());
		}
	}
}
 

/**
 * Tests that each checkpoint is only aborted once in case of an interleaved cancellation
 * barrier arrival of two consecutive checkpoints.
 */
@Test
public void testInterleavedCancellationBarriers() throws Exception {
	BufferOrEvent[] sequence = {
		createBarrier(1L, 0),
		createCancellationBarrier(2L, 0),
		createCancellationBarrier(1L, 1),
		createCancellationBarrier(2L, 1),
		createCancellationBarrier(1L, 2),
		createCancellationBarrier(2L, 2),
		createBuffer(0)
	};
	CheckpointSequenceValidator validator =
		new CheckpointSequenceValidator(-1, -2);
	inputGate = createBarrierTracker(3, sequence, validator);

	for (BufferOrEvent boe : sequence) {
		if (boe.isBuffer() || (boe.getEvent().getClass() != CheckpointBarrier.class && boe.getEvent().getClass() != CancelCheckpointMarker.class)) {
			assertEquals(boe, inputGate.pollNext().get());
		}
	}
}
 

private void verifyBroadcastEvent(
		RecordWriterDelegate writerDelegate,
		ArrayDeque<BufferConsumer>[] queues,
		int numRecordWriters) throws Exception {

	final CancelCheckpointMarker message = new CancelCheckpointMarker(1);
	writerDelegate.broadcastEvent(message);

	// verify the added messages in all the queues
	for (int i = 0; i < queues.length; i++) {
		assertEquals(numRecordWriters, queues[i].size());

		for (int j = 0; j < numRecordWriters; j++) {
			BufferOrEvent boe = RecordWriterTest.parseBuffer(queues[i].remove(), i);
			assertTrue(boe.isEvent());
			assertEquals(message, boe.getEvent());
		}
	}
}
 

@Override
public void abortCheckpointOnBarrier(long checkpointId, Throwable cause, OperatorChain<?, ?> operatorChain) throws IOException {
	LOG.debug("Aborting checkpoint via cancel-barrier {} for task {}", checkpointId, taskName);
	lastCheckpointId = Math.max(lastCheckpointId, checkpointId);
	Iterator<Long> iterator = abortedCheckpointIds.iterator();
	while (iterator.hasNext()) {
		long next = iterator.next();
		if (next < lastCheckpointId) {
			iterator.remove();
		} else {
			break;
		}
	}

	checkpointStorage.clearCacheFor(checkpointId);

	channelStateWriter.abort(checkpointId, cause, true);

	// notify the coordinator that we decline this checkpoint
	env.declineCheckpoint(checkpointId, cause);

	// notify all downstream operators that they should not wait for a barrier from us
	actionExecutor.runThrowing(() -> operatorChain.broadcastEvent(new CancelCheckpointMarker(checkpointId)));
}
 

/**
 * This test checks that tasks emit a proper cancel checkpoint barrier, if a "trigger checkpoint" message
 * comes before they are ready.
 */
@Test
public void testEmitCancellationBarrierWhenNotReady() throws Exception {
	StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>(
			InitBlockingTask::new, BasicTypeInfo.STRING_TYPE_INFO);
	testHarness.setupOutputForSingletonOperatorChain();

	// start the test - this cannot succeed across the 'init()' method
	testHarness.invoke();

	StreamTask<String, ?> task = testHarness.getTask();

	// tell the task to commence a checkpoint
	boolean result = task.triggerCheckpoint(new CheckpointMetaData(41L, System.currentTimeMillis()),
		CheckpointOptions.forCheckpointWithDefaultLocation());
	assertFalse("task triggered checkpoint though not ready", result);

	// a cancellation barrier should be downstream
	Object emitted = testHarness.getOutput().poll();
	assertNotNull("nothing emitted", emitted);
	assertTrue("wrong type emitted", emitted instanceof CancelCheckpointMarker);
	assertEquals("wrong checkpoint id", 41L, ((CancelCheckpointMarker) emitted).getCheckpointId());
}
 

@Override
public BufferOrEvent getNextNonBlocked() throws Exception {
	while (true) {
		Optional<BufferOrEvent> next = inputGate.getNextBufferOrEvent();
		if (!next.isPresent()) {
			// buffer or input exhausted
			return null;
		}

		BufferOrEvent bufferOrEvent = next.get();
		if (bufferOrEvent.isBuffer()) {
			return bufferOrEvent;
		}
		else if (bufferOrEvent.getEvent().getClass() == CheckpointBarrier.class) {
			processBarrier((CheckpointBarrier) bufferOrEvent.getEvent(), bufferOrEvent.getChannelIndex());
		}
		else if (bufferOrEvent.getEvent().getClass() == CancelCheckpointMarker.class) {
			processCheckpointAbortBarrier((CancelCheckpointMarker) bufferOrEvent.getEvent(), bufferOrEvent.getChannelIndex());
		}
		else {
			// some other event
			return bufferOrEvent;
		}
	}
}
 

@Test
public void testMissingCancellationBarriers() throws Exception {
	BufferOrEvent[] sequence = {
		createBarrier(1L, 0),
		createCancellationBarrier(3L, 1),
		createCancellationBarrier(2L, 0),
		createCancellationBarrier(3L, 0),
		createBuffer(0)
	};
	AbstractInvokable validator = new ValidatingCheckpointHandler();
	inputGate = createBarrierBuffer(2, sequence, validator);

	for (BufferOrEvent boe : sequence) {
		if (boe.isBuffer() || boe.getEvent().getClass() != CancelCheckpointMarker.class) {
			assertEquals(boe, inputGate.pollNext().get());
		}
	}

	Integer[] expectedUnblockedChannels = new Integer[] {0};
	assertArrayEquals(expectedUnblockedChannels, mockInputGate.getAndResetLastUnblockedChannels().toArray());
}
 

@Test
public void testSingleChannelAbortCheckpoint() throws Exception {
	BufferOrEvent[] sequence = {
			createBuffer(0),
			createBarrier(1, 0),
			createBuffer(0),
			createBarrier(2, 0),
			createCancellationBarrier(4, 0),
			createBarrier(5, 0),
			createBuffer(0),
			createCancellationBarrier(6, 0),
			createBuffer(0)
	};
	// negative values mean an expected cancellation call!
	CheckpointSequenceValidator validator =
		new CheckpointSequenceValidator(1, 2, -4, 5, -6);
	inputGate = createBarrierTracker(1, sequence, validator);

	for (BufferOrEvent boe : sequence) {
		if (boe.isBuffer() || boe.getEvent().getClass() != CancelCheckpointMarker.class) {
			assertEquals(boe, inputGate.pollNext().get());
		}
	}
}
 

/**
 * Tests that each checkpoint is only aborted once in case of an interleaved cancellation
 * barrier arrival of two consecutive checkpoints.
 */
@Test
public void testInterleavedCancellationBarriers() throws Exception {
	BufferOrEvent[] sequence = {
		createBarrier(1L, 0),
		createCancellationBarrier(2L, 0),
		createCancellationBarrier(1L, 1),
		createCancellationBarrier(2L, 1),
		createCancellationBarrier(1L, 2),
		createCancellationBarrier(2L, 2),
		createBuffer(0)
	};
	CheckpointSequenceValidator validator =
		new CheckpointSequenceValidator(-1, -2);
	inputGate = createBarrierTracker(3, sequence, validator);

	for (BufferOrEvent boe : sequence) {
		if (boe.isBuffer() || boe.getEvent().getClass() != CancelCheckpointMarker.class) {
			assertEquals(boe, inputGate.pollNext().get());
		}
	}
}
 

@Test
public void testSerializeDeserializeEvent() throws Exception {
	AbstractEvent[] events = {
			EndOfPartitionEvent.INSTANCE,
			EndOfSuperstepEvent.INSTANCE,
			new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forCheckpointWithDefaultLocation()),
			new TestTaskEvent(Math.random(), 12361231273L),
			new CancelCheckpointMarker(287087987329842L)
	};

	for (AbstractEvent evt : events) {
		ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
		assertTrue(serializedEvent.hasRemaining());

		AbstractEvent deserialized =
				EventSerializer.fromSerializedEvent(serializedEvent, getClass().getClassLoader());
		assertNotNull(deserialized);
		assertEquals(evt, deserialized);
	}
}
 

private void testProcessCancellationBarrier(
		CheckpointBarrierUnaligner handler,
		ValidatingCheckpointInvokable invokable) throws Exception {

	final long cancelledCheckpointId = new Random().nextBoolean() ? DEFAULT_CHECKPOINT_ID : DEFAULT_CHECKPOINT_ID + 1L;
	// should abort current checkpoint while processing CancelCheckpointMarker
	handler.processCancellationBarrier(new CancelCheckpointMarker(cancelledCheckpointId));
	verifyTriggeredCheckpoint(handler, invokable, cancelledCheckpointId);

	final long nextCancelledCheckpointId = cancelledCheckpointId + 1L;
	// should update current checkpoint id and abort notification while processing CancelCheckpointMarker
	handler.processCancellationBarrier(new CancelCheckpointMarker(nextCancelledCheckpointId));
	verifyTriggeredCheckpoint(handler, invokable, nextCancelledCheckpointId);
}
 

/**
 * Tests that each checkpoint is only aborted once in case of an interleaved cancellation
 * barrier arrival of two consecutive checkpoints.
 */
@Test
public void testInterleavedCancellationBarriers() throws Exception {
	BufferOrEvent[] sequence = {
		createBarrier(1L, 0),
		createCancellationBarrier(2L, 0),
		createCancellationBarrier(1L, 1),
		createCancellationBarrier(2L, 1),
		createCancellationBarrier(1L, 2),
		createCancellationBarrier(2L, 2),
		createBuffer(0)
	};

	MockInputGate gate = new MockInputGate(PAGE_SIZE, 3, Arrays.asList(sequence));
	BarrierTracker tracker = new BarrierTracker(gate);
	AbstractInvokable statefulTask = mock(AbstractInvokable.class);

	tracker.registerCheckpointEventHandler(statefulTask);

	for (BufferOrEvent boe : sequence) {
		if (boe.isBuffer() || (boe.getEvent().getClass() != CheckpointBarrier.class && boe.getEvent().getClass() != CancelCheckpointMarker.class)) {
			assertEquals(boe, tracker.getNextNonBlocked());
		}
	}

	verify(statefulTask, times(1)).abortCheckpointOnBarrier(eq(1L), any(Throwable.class));
	verify(statefulTask, times(1)).abortCheckpointOnBarrier(eq(2L), any(Throwable.class));
}
 

/**
 * Tests {@link EventSerializer#isEvent(Buffer, Class)} returns
 * the correct answer for various encoded event buffers.
 */
@Test
public void testIsEvent() throws Exception {
	AbstractEvent[] events = {
		EndOfPartitionEvent.INSTANCE,
		EndOfSuperstepEvent.INSTANCE,
		new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forCheckpointWithDefaultLocation()),
		new TestTaskEvent(Math.random(), 12361231273L),
		new CancelCheckpointMarker(287087987329842L)
	};

	Class[] expectedClasses = Arrays.stream(events)
		.map(AbstractEvent::getClass)
		.toArray(Class[]::new);

	for (AbstractEvent evt : events) {
		for (Class<?> expectedClass: expectedClasses) {
			if (expectedClass.equals(TestTaskEvent.class)) {
				try {
					checkIsEvent(evt, expectedClass);
					fail("This should fail");
				}
				catch (UnsupportedOperationException ex) {
					// expected
				}
			}
			else if (evt.getClass().equals(expectedClass)) {
				assertTrue(checkIsEvent(evt, expectedClass));
			} else {
				assertFalse(checkIsEvent(evt, expectedClass));
			}
		}
	}
}
 

/**
 * Identifies whether the given buffer encodes the given event. Custom events are not supported.
 *
 * <p><strong>Pre-condition</strong>: This buffer must encode some event!</p>
 *
 * @param buffer the buffer to peak into
 * @param eventClass the expected class of the event type
 * @return whether the event class of the <tt>buffer</tt> matches the given <tt>eventClass</tt>
 */
private static boolean isEvent(ByteBuffer buffer, Class<?> eventClass) throws IOException {
	if (buffer.remaining() < 4) {
		throw new IOException("Incomplete event");
	}

	final int bufferPos = buffer.position();
	final ByteOrder bufferOrder = buffer.order();
	buffer.order(ByteOrder.BIG_ENDIAN);

	try {
		int type = buffer.getInt();

		if (eventClass.equals(EndOfPartitionEvent.class)) {
			return type == END_OF_PARTITION_EVENT;
		} else if (eventClass.equals(CheckpointBarrier.class)) {
			return type == CHECKPOINT_BARRIER_EVENT;
		} else if (eventClass.equals(EndOfSuperstepEvent.class)) {
			return type == END_OF_SUPERSTEP_EVENT;
		} else if (eventClass.equals(CancelCheckpointMarker.class)) {
			return type == CANCEL_CHECKPOINT_MARKER_EVENT;
		} else {
			throw new UnsupportedOperationException("Unsupported eventClass = " + eventClass);
		}
	}
	finally {
		buffer.order(bufferOrder);
		// restore the original position in the buffer (recall: we only peak into it!)
		buffer.position(bufferPos);
	}
}
 

/**
 * This test verifies (for two input tasks) that the Stream tasks react the following way to
 * receiving a checkpoint cancellation barrier:
 *   - send a "decline checkpoint" notification out (to the JobManager)
 *   - emit a cancellation barrier downstream.
 */
@Test
public void testDeclineCallOnCancelBarrierTwoInputs() throws Exception {

	TwoInputStreamTaskTestHarness<String, String, String> testHarness = new TwoInputStreamTaskTestHarness<>(
			TwoInputStreamTask::new,
			BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	CoStreamMap<String, String, String> op = new CoStreamMap<>(new UnionCoMap());
	streamConfig.setStreamOperator(op);
	streamConfig.setOperatorID(new OperatorID());

	StreamMockEnvironment environment = spy(testHarness.createEnvironment());

	// start the task
	testHarness.invoke(environment);
	testHarness.waitForTaskRunning();

	// emit cancellation barriers
	testHarness.processEvent(new CancelCheckpointMarker(2L), 0, 0);
	testHarness.processEvent(new CancelCheckpointMarker(2L), 1, 0);
	testHarness.waitForInputProcessing();

	// the decline call should go to the coordinator
	verify(environment, times(1)).declineCheckpoint(eq(2L), any(CheckpointDeclineOnCancellationBarrierException.class));

	// a cancellation barrier should be downstream
	Object result = testHarness.getOutput().poll();
	assertNotNull("nothing emitted", result);
	assertTrue("wrong type emitted", result instanceof CancelCheckpointMarker);
	assertEquals("wrong checkpoint id", 2L, ((CancelCheckpointMarker) result).getCheckpointId());

	// cancel and shutdown
	testHarness.endInput();
	testHarness.waitForTaskCompletion();
}
 

public static ByteBuffer toSerializedEvent(AbstractEvent event) throws IOException {
	final Class<?> eventClass = event.getClass();
	if (eventClass == EndOfPartitionEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_PARTITION_EVENT });
	}
	else if (eventClass == CheckpointBarrier.class) {
		return serializeCheckpointBarrier((CheckpointBarrier) event);
	}
	else if (eventClass == EndOfSuperstepEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_SUPERSTEP_EVENT });
	}
	else if (eventClass == CancelCheckpointMarker.class) {
		CancelCheckpointMarker marker = (CancelCheckpointMarker) event;

		ByteBuffer buf = ByteBuffer.allocate(12);
		buf.putInt(0, CANCEL_CHECKPOINT_MARKER_EVENT);
		buf.putLong(4, marker.getCheckpointId());
		return buf;
	}
	else {
		try {
			final DataOutputSerializer serializer = new DataOutputSerializer(128);
			serializer.writeInt(OTHER_EVENT);
			serializer.writeUTF(event.getClass().getName());
			event.write(serializer);
			return serializer.wrapAsByteBuffer();
		}
		catch (IOException e) {
			throw new IOException("Error while serializing event.", e);
		}
	}
}
 

/**
 * Tests {@link EventSerializer#isEvent(Buffer, Class)} returns
 * the correct answer for various encoded event buffers.
 */
@Test
public void testIsEvent() throws Exception {
	AbstractEvent[] events = {
		EndOfPartitionEvent.INSTANCE,
		EndOfSuperstepEvent.INSTANCE,
		new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forCheckpointWithDefaultLocation()),
		new TestTaskEvent(Math.random(), 12361231273L),
		new CancelCheckpointMarker(287087987329842L)
	};

	Class[] expectedClasses = Arrays.stream(events)
		.map(AbstractEvent::getClass)
		.toArray(Class[]::new);

	for (AbstractEvent evt : events) {
		for (Class<?> expectedClass: expectedClasses) {
			if (expectedClass.equals(TestTaskEvent.class)) {
				try {
					checkIsEvent(evt, expectedClass);
					fail("This should fail");
				}
				catch (UnsupportedOperationException ex) {
					// expected
				}
			}
			else if (evt.getClass().equals(expectedClass)) {
				assertTrue(checkIsEvent(evt, expectedClass));
			} else {
				assertFalse(checkIsEvent(evt, expectedClass));
			}
		}
	}
}
 

/**
 * Identifies whether the given buffer encodes the given event. Custom events are not supported.
 *
 * <p><strong>Pre-condition</strong>: This buffer must encode some event!</p>
 *
 * @param buffer the buffer to peak into
 * @param eventClass the expected class of the event type
 * @return whether the event class of the <tt>buffer</tt> matches the given <tt>eventClass</tt>
 */
private static boolean isEvent(ByteBuffer buffer, Class<?> eventClass) throws IOException {
	if (buffer.remaining() < 4) {
		throw new IOException("Incomplete event");
	}

	final int bufferPos = buffer.position();
	final ByteOrder bufferOrder = buffer.order();
	buffer.order(ByteOrder.BIG_ENDIAN);

	try {
		int type = buffer.getInt();

		if (eventClass.equals(EndOfPartitionEvent.class)) {
			return type == END_OF_PARTITION_EVENT;
		} else if (eventClass.equals(CheckpointBarrier.class)) {
			return type == CHECKPOINT_BARRIER_EVENT;
		} else if (eventClass.equals(EndOfSuperstepEvent.class)) {
			return type == END_OF_SUPERSTEP_EVENT;
		} else if (eventClass.equals(CancelCheckpointMarker.class)) {
			return type == CANCEL_CHECKPOINT_MARKER_EVENT;
		} else {
			throw new UnsupportedOperationException("Unsupported eventClass = " + eventClass);
		}
	}
	finally {
		buffer.order(bufferOrder);
		// restore the original position in the buffer (recall: we only peak into it!)
		buffer.position(bufferPos);
	}
}
 

/**
 * This test verifies (for onw input tasks) that the Stream tasks react the following way to
 * receiving a checkpoint cancellation barrier:
 *   - send a "decline checkpoint" notification out (to the JobManager)
 *   - emit a cancellation barrier downstream.
 */
@Test
public void testDeclineCallOnCancelBarrierOneInput() throws Exception {

	OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			1, 2,
			BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	StreamMap<String, String> mapOperator = new StreamMap<>(new IdentityMap());
	streamConfig.setStreamOperator(mapOperator);
	streamConfig.setOperatorID(new OperatorID());

	StreamMockEnvironment environment = spy(testHarness.createEnvironment());

	// start the task
	testHarness.invoke(environment);
	testHarness.waitForTaskRunning();

	// emit cancellation barriers
	testHarness.processEvent(new CancelCheckpointMarker(2L), 0, 1);
	testHarness.processEvent(new CancelCheckpointMarker(2L), 0, 0);
	testHarness.waitForInputProcessing();

	// the decline call should go to the coordinator
	verify(environment, times(1)).declineCheckpoint(eq(2L),
		argThat(new CheckpointExceptionMatcher(CheckpointFailureReason.CHECKPOINT_DECLINED_ON_CANCELLATION_BARRIER)));

	// a cancellation barrier should be downstream
	Object result = testHarness.getOutput().poll();
	assertNotNull("nothing emitted", result);
	assertTrue("wrong type emitted", result instanceof CancelCheckpointMarker);
	assertEquals("wrong checkpoint id", 2L, ((CancelCheckpointMarker) result).getCheckpointId());

	// cancel and shutdown
	testHarness.endInput();
	testHarness.waitForTaskCompletion();
}
 

/**
 * This test verifies (for two input tasks) that the Stream tasks react the following way to
 * receiving a checkpoint cancellation barrier:
 *   - send a "decline checkpoint" notification out (to the JobManager)
 *   - emit a cancellation barrier downstream.
 */
@Test
public void testDeclineCallOnCancelBarrierTwoInputs() throws Exception {

	TwoInputStreamTaskTestHarness<String, String, String> testHarness = new TwoInputStreamTaskTestHarness<>(
			TwoInputStreamTask::new,
			BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	CoStreamMap<String, String, String> op = new CoStreamMap<>(new UnionCoMap());
	streamConfig.setStreamOperator(op);
	streamConfig.setOperatorID(new OperatorID());

	StreamMockEnvironment environment = spy(testHarness.createEnvironment());

	// start the task
	testHarness.invoke(environment);
	testHarness.waitForTaskRunning();

	// emit cancellation barriers
	testHarness.processEvent(new CancelCheckpointMarker(2L), 0, 0);
	testHarness.processEvent(new CancelCheckpointMarker(2L), 1, 0);
	testHarness.waitForInputProcessing();

	// the decline call should go to the coordinator
	verify(environment, times(1)).declineCheckpoint(eq(2L),
		argThat(new CheckpointBarrierAlignerTestBase.CheckpointExceptionMatcher(CheckpointFailureReason.CHECKPOINT_DECLINED_ON_CANCELLATION_BARRIER)));

	// a cancellation barrier should be downstream
	Object result = testHarness.getOutput().poll();
	assertNotNull("nothing emitted", result);
	assertTrue("wrong type emitted", result instanceof CancelCheckpointMarker);
	assertEquals("wrong checkpoint id", 2L, ((CancelCheckpointMarker) result).getCheckpointId());

	// cancel and shutdown
	testHarness.endInput();
	testHarness.waitForTaskCompletion();
}
 

/**
 * This test verifies (for two input tasks) that the Stream tasks react the following way to
 * receiving a checkpoint cancellation barrier:
 *   - send a "decline checkpoint" notification out (to the JobManager)
 *   - emit a cancellation barrier downstream.
 */
@Test
public void testDeclineCallOnCancelBarrierTwoInputs() throws Exception {

	TwoInputStreamTaskTestHarness<String, String, String> testHarness = new TwoInputStreamTaskTestHarness<>(
			TwoInputStreamTask::new,
			BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	CoStreamMap<String, String, String> op = new CoStreamMap<>(new UnionCoMap());
	streamConfig.setStreamOperator(op);
	streamConfig.setOperatorID(new OperatorID());

	StreamMockEnvironment environment = spy(testHarness.createEnvironment());

	// start the task
	testHarness.invoke(environment);
	testHarness.waitForTaskRunning();

	// emit cancellation barriers
	testHarness.processEvent(new CancelCheckpointMarker(2L), 0, 0);
	testHarness.processEvent(new CancelCheckpointMarker(2L), 1, 0);
	testHarness.waitForInputProcessing();

	// the decline call should go to the coordinator
	verify(environment, times(1)).declineCheckpoint(eq(2L),
		argThat(new CheckpointBarrierAlignerTestBase.CheckpointExceptionMatcher(CheckpointFailureReason.CHECKPOINT_DECLINED_ON_CANCELLATION_BARRIER)));

	// a cancellation barrier should be downstream
	Object result = testHarness.getOutput().poll();
	assertNotNull("nothing emitted", result);
	assertTrue("wrong type emitted", result instanceof CancelCheckpointMarker);
	assertEquals("wrong checkpoint id", 2L, ((CancelCheckpointMarker) result).getCheckpointId());

	// cancel and shutdown
	testHarness.endInput();
	testHarness.waitForTaskCompletion();
}
 

@Test
public void testProcessCancellationBarrierBeforeProcessAndReceiveBarrier() throws Exception {
	final ValidatingCheckpointInvokable invokable = new ValidatingCheckpointInvokable();
	final CheckpointBarrierUnaligner handler = new CheckpointBarrierUnaligner(TestSubtaskCheckpointCoordinator.INSTANCE, "test", invokable, new MockIndexedInputGate());

	handler.processCancellationBarrier(new CancelCheckpointMarker(DEFAULT_CHECKPOINT_ID));

	verifyTriggeredCheckpoint(handler, invokable, DEFAULT_CHECKPOINT_ID);

	// it would not trigger checkpoint since the respective cancellation barrier already happened before
	handler.processBarrier(buildCheckpointBarrier(DEFAULT_CHECKPOINT_ID), new InputChannelInfo(0, 0));
	handler.getThreadSafeUnaligner().notifyBarrierReceived(buildCheckpointBarrier(DEFAULT_CHECKPOINT_ID), new InputChannelInfo(0, 0));

	verifyTriggeredCheckpoint(handler, invokable, DEFAULT_CHECKPOINT_ID);
}
 

public static ByteBuffer toSerializedEvent(AbstractEvent event) throws IOException {
	final Class<?> eventClass = event.getClass();
	if (eventClass == EndOfPartitionEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_PARTITION_EVENT });
	}
	else if (eventClass == CheckpointBarrier.class) {
		return serializeCheckpointBarrier((CheckpointBarrier) event);
	}
	else if (eventClass == EndOfSuperstepEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_SUPERSTEP_EVENT });
	}
	else if (eventClass == EndOfChannelStateEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_CHANNEL_STATE_EVENT });
	}
	else if (eventClass == CancelCheckpointMarker.class) {
		CancelCheckpointMarker marker = (CancelCheckpointMarker) event;

		ByteBuffer buf = ByteBuffer.allocate(12);
		buf.putInt(0, CANCEL_CHECKPOINT_MARKER_EVENT);
		buf.putLong(4, marker.getCheckpointId());
		return buf;
	}
	else {
		try {
			final DataOutputSerializer serializer = new DataOutputSerializer(128);
			serializer.writeInt(OTHER_EVENT);
			serializer.writeUTF(event.getClass().getName());
			event.write(serializer);
			return serializer.wrapAsByteBuffer();
		}
		catch (IOException e) {
			throw new IOException("Error while serializing event.", e);
		}
	}
}
 

/**
 * Identifies whether the given buffer encodes the given event. Custom events are not supported.
 *
 * <p><strong>Pre-condition</strong>: This buffer must encode some event!</p>
 *
 * @param buffer the buffer to peak into
 * @param eventClass the expected class of the event type
 * @return whether the event class of the <tt>buffer</tt> matches the given <tt>eventClass</tt>
 */
private static boolean isEvent(ByteBuffer buffer, Class<?> eventClass) throws IOException {
	if (buffer.remaining() < 4) {
		throw new IOException("Incomplete event");
	}

	final int bufferPos = buffer.position();
	final ByteOrder bufferOrder = buffer.order();
	buffer.order(ByteOrder.BIG_ENDIAN);

	try {
		int type = buffer.getInt();

		if (eventClass.equals(EndOfPartitionEvent.class)) {
			return type == END_OF_PARTITION_EVENT;
		} else if (eventClass.equals(CheckpointBarrier.class)) {
			return type == CHECKPOINT_BARRIER_EVENT;
		} else if (eventClass.equals(EndOfSuperstepEvent.class)) {
			return type == END_OF_SUPERSTEP_EVENT;
		} else if (eventClass.equals(EndOfChannelStateEvent.class)) {
			return type == END_OF_CHANNEL_STATE_EVENT;
		} else if (eventClass.equals(CancelCheckpointMarker.class)) {
			return type == CANCEL_CHECKPOINT_MARKER_EVENT;
		} else {
			throw new UnsupportedOperationException("Unsupported eventClass = " + eventClass);
		}
	}
	finally {
		buffer.order(bufferOrder);
		// restore the original position in the buffer (recall: we only peak into it!)
		buffer.position(bufferPos);
	}
}
 

@Override
public Optional<BufferOrEvent> pollNext() throws Exception {
	while (true) {
		Optional<BufferOrEvent> next = inputGate.pollNext();

		if (!next.isPresent()) {
			return handleEmptyBuffer();
		}

		BufferOrEvent bufferOrEvent = next.get();
		checkState(!barrierHandler.isBlocked(bufferOrEvent.getChannelInfo()));

		if (bufferOrEvent.isBuffer()) {
			return next;
		}
		else if (bufferOrEvent.getEvent().getClass() == CheckpointBarrier.class) {
			CheckpointBarrier checkpointBarrier = (CheckpointBarrier) bufferOrEvent.getEvent();
			barrierHandler.processBarrier(checkpointBarrier, bufferOrEvent.getChannelInfo());
			return next;
		}
		else if (bufferOrEvent.getEvent().getClass() == CancelCheckpointMarker.class) {
			barrierHandler.processCancellationBarrier((CancelCheckpointMarker) bufferOrEvent.getEvent());
		}
		else {
			if (bufferOrEvent.getEvent().getClass() == EndOfPartitionEvent.class) {
				barrierHandler.processEndOfPartition();
			}
			return next;
		}
	}
}
 

@Override
public void processCancellationBarrier(CancelCheckpointMarker cancelBarrier) throws Exception {
	final long cancelledId = cancelBarrier.getCheckpointId();
	boolean shouldAbort = threadSafeUnaligner.setCancelledCheckpointId(cancelledId);
	if (shouldAbort) {
		notifyAbort(
			cancelledId,
			new CheckpointException(CheckpointFailureReason.CHECKPOINT_DECLINED_ON_CANCELLATION_BARRIER));
	}

	if (cancelledId >= currentConsumedCheckpointId) {
		resetPendingCheckpoint(cancelledId);
		currentConsumedCheckpointId = cancelledId;
	}
}
 

/**
 * This test verifies (for onw input tasks) that the Stream tasks react the following way to
 * receiving a checkpoint cancellation barrier:
 *   - send a "decline checkpoint" notification out (to the JobManager)
 *   - emit a cancellation barrier downstream.
 */
@Test
public void testDeclineCallOnCancelBarrierOneInput() throws Exception {

	OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			1, 2,
			BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	StreamMap<String, String> mapOperator = new StreamMap<>(new IdentityMap());
	streamConfig.setStreamOperator(mapOperator);
	streamConfig.setOperatorID(new OperatorID());

	StreamMockEnvironment environment = spy(testHarness.createEnvironment());

	// start the task
	testHarness.invoke(environment);
	testHarness.waitForTaskRunning();

	// emit cancellation barriers
	testHarness.processEvent(new CancelCheckpointMarker(2L), 0, 1);
	testHarness.processEvent(new CancelCheckpointMarker(2L), 0, 0);
	testHarness.waitForInputProcessing();

	// the decline call should go to the coordinator
	verify(environment, times(1)).declineCheckpoint(eq(2L),
		argThat(new CheckpointExceptionMatcher(CheckpointFailureReason.CHECKPOINT_DECLINED_ON_CANCELLATION_BARRIER)));

	// a cancellation barrier should be downstream
	Object result = testHarness.getOutput().poll();
	assertNotNull("nothing emitted", result);
	assertTrue("wrong type emitted", result instanceof CancelCheckpointMarker);
	assertEquals("wrong checkpoint id", 2L, ((CancelCheckpointMarker) result).getCheckpointId());

	// cancel and shutdown
	testHarness.endInput();
	testHarness.waitForTaskCompletion();
}