org.apache.flink.runtime.io.network.api.EndOfSuperstepEvent Java Examples

@Test
public void testSerializeDeserializeEvent() throws Exception {
	AbstractEvent[] events = {
			EndOfPartitionEvent.INSTANCE,
			EndOfSuperstepEvent.INSTANCE,
			new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forCheckpointWithDefaultLocation()),
			new TestTaskEvent(Math.random(), 12361231273L),
			new CancelCheckpointMarker(287087987329842L)
	};

	for (AbstractEvent evt : events) {
		ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
		assertTrue(serializedEvent.hasRemaining());

		AbstractEvent deserialized =
				EventSerializer.fromSerializedEvent(serializedEvent, getClass().getClassLoader());
		assertNotNull(deserialized);
		assertEquals(evt, deserialized);
	}
}
 

@Test
public void testSerializeDeserializeEvent() throws Exception {
	AbstractEvent[] events = {
			EndOfPartitionEvent.INSTANCE,
			EndOfSuperstepEvent.INSTANCE,
			new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forCheckpointWithDefaultLocation()),
			new TestTaskEvent(Math.random(), 12361231273L),
			new CancelCheckpointMarker(287087987329842L)
	};

	for (AbstractEvent evt : events) {
		ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
		assertTrue(serializedEvent.hasRemaining());

		AbstractEvent deserialized =
				EventSerializer.fromSerializedEvent(serializedEvent, getClass().getClassLoader());
		assertNotNull(deserialized);
		assertEquals(evt, deserialized);
	}
}
 

/**
 * Identifies whether the given buffer encodes the given event. Custom events are not supported.
 *
 * <p><strong>Pre-condition</strong>: This buffer must encode some event!</p>
 *
 * @param buffer the buffer to peak into
 * @param eventClass the expected class of the event type
 * @return whether the event class of the <tt>buffer</tt> matches the given <tt>eventClass</tt>
 */
private static boolean isEvent(ByteBuffer buffer, Class<?> eventClass) throws IOException {
	if (buffer.remaining() < 4) {
		throw new IOException("Incomplete event");
	}

	final int bufferPos = buffer.position();
	final ByteOrder bufferOrder = buffer.order();
	buffer.order(ByteOrder.BIG_ENDIAN);

	try {
		int type = buffer.getInt();

		if (eventClass.equals(EndOfPartitionEvent.class)) {
			return type == END_OF_PARTITION_EVENT;
		} else if (eventClass.equals(CheckpointBarrier.class)) {
			return type == CHECKPOINT_BARRIER_EVENT;
		} else if (eventClass.equals(EndOfSuperstepEvent.class)) {
			return type == END_OF_SUPERSTEP_EVENT;
		} else if (eventClass.equals(CancelCheckpointMarker.class)) {
			return type == CANCEL_CHECKPOINT_MARKER_EVENT;
		} else {
			throw new UnsupportedOperationException("Unsupported eventClass = " + eventClass);
		}
	}
	finally {
		buffer.order(bufferOrder);
		// restore the original position in the buffer (recall: we only peak into it!)
		buffer.position(bufferPos);
	}
}
 

@Test
public void testBarrierOvertaking() throws Exception {
	subpartition.add(createFilledFinishedBufferConsumer(1));
	assertEquals(0, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	subpartition.add(createFilledFinishedBufferConsumer(2));
	assertEquals(1, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	BufferConsumer eventBuffer = EventSerializer.toBufferConsumer(EndOfSuperstepEvent.INSTANCE);
	subpartition.add(eventBuffer);
	assertEquals(1, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	subpartition.add(createFilledFinishedBufferConsumer(4));
	assertEquals(1, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	CheckpointOptions options = new CheckpointOptions(
		CheckpointType.CHECKPOINT,
		new CheckpointStorageLocationReference(new byte[]{0, 1, 2}),
		true,
		true);
	BufferConsumer barrierBuffer = EventSerializer.toBufferConsumer(new CheckpointBarrier(0, 0, options));
	subpartition.add(barrierBuffer, true);
	assertEquals(2, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	List<Buffer> inflight = subpartition.requestInflightBufferSnapshot();
	assertEquals(Arrays.asList(1, 2, 4), inflight.stream().map(Buffer::getSize).collect(Collectors.toList()));
	inflight.forEach(Buffer::recycleBuffer);

	assertNextEvent(readView, barrierBuffer.getWrittenBytes(), CheckpointBarrier.class, true, 2, false, true);
	assertNextBuffer(readView, 1, true, 1, false, true);
	assertNextBuffer(readView, 2, true, 0, true, true);
	assertNextEvent(readView, eventBuffer.getWrittenBytes(), EndOfSuperstepEvent.class, false, 0, false, true);
	assertNextBuffer(readView, 4, false, 0, false, true);
	assertNoNextBuffer(readView);
}
 

/**
 * Handles the event and returns whether the reader reached an end-of-stream event (either the
 * end of the whole stream or the end of an superstep).
 */
protected boolean handleEvent(AbstractEvent event) throws IOException {
	final Class<?> eventType = event.getClass();

	try {
		// ------------------------------------------------------------
		// Runtime events
		// ------------------------------------------------------------

		// This event is also checked at the (single) input gate to release the respective
		// channel, at which it was received.
		if (eventType == EndOfPartitionEvent.class) {
			return true;
		}
		else if (eventType == EndOfSuperstepEvent.class) {
			return incrementEndOfSuperstepEventAndCheck();
		}

		// ------------------------------------------------------------
		// Task events (user)
		// ------------------------------------------------------------
		else if (event instanceof TaskEvent) {
			taskEventHandler.publish((TaskEvent) event);

			return false;
		}
		else {
			throw new IllegalStateException("Received unexpected event of type " + eventType + " at reader.");
		}
	}
	catch (Throwable t) {
		throw new IOException("Error while handling event of type " + eventType + ": " + t.getMessage(), t);
	}
}
 

/**
 * Identifies whether the given buffer encodes the given event. Custom events are not supported.
 *
 * <p><strong>Pre-condition</strong>: This buffer must encode some event!</p>
 *
 * @param buffer the buffer to peak into
 * @param eventClass the expected class of the event type
 * @return whether the event class of the <tt>buffer</tt> matches the given <tt>eventClass</tt>
 */
private static boolean isEvent(ByteBuffer buffer, Class<?> eventClass) throws IOException {
	if (buffer.remaining() < 4) {
		throw new IOException("Incomplete event");
	}

	final int bufferPos = buffer.position();
	final ByteOrder bufferOrder = buffer.order();
	buffer.order(ByteOrder.BIG_ENDIAN);

	try {
		int type = buffer.getInt();

		if (eventClass.equals(EndOfPartitionEvent.class)) {
			return type == END_OF_PARTITION_EVENT;
		} else if (eventClass.equals(CheckpointBarrier.class)) {
			return type == CHECKPOINT_BARRIER_EVENT;
		} else if (eventClass.equals(EndOfSuperstepEvent.class)) {
			return type == END_OF_SUPERSTEP_EVENT;
		} else if (eventClass.equals(EndOfChannelStateEvent.class)) {
			return type == END_OF_CHANNEL_STATE_EVENT;
		} else if (eventClass.equals(CancelCheckpointMarker.class)) {
			return type == CANCEL_CHECKPOINT_MARKER_EVENT;
		} else {
			throw new UnsupportedOperationException("Unsupported eventClass = " + eventClass);
		}
	}
	finally {
		buffer.order(bufferOrder);
		// restore the original position in the buffer (recall: we only peak into it!)
		buffer.position(bufferPos);
	}
}
 

public static ByteBuffer toSerializedEvent(AbstractEvent event) throws IOException {
	final Class<?> eventClass = event.getClass();
	if (eventClass == EndOfPartitionEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_PARTITION_EVENT });
	}
	else if (eventClass == CheckpointBarrier.class) {
		return serializeCheckpointBarrier((CheckpointBarrier) event);
	}
	else if (eventClass == EndOfSuperstepEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_SUPERSTEP_EVENT });
	}
	else if (eventClass == EndOfChannelStateEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_CHANNEL_STATE_EVENT });
	}
	else if (eventClass == CancelCheckpointMarker.class) {
		CancelCheckpointMarker marker = (CancelCheckpointMarker) event;

		ByteBuffer buf = ByteBuffer.allocate(12);
		buf.putInt(0, CANCEL_CHECKPOINT_MARKER_EVENT);
		buf.putLong(4, marker.getCheckpointId());
		return buf;
	}
	else {
		try {
			final DataOutputSerializer serializer = new DataOutputSerializer(128);
			serializer.writeInt(OTHER_EVENT);
			serializer.writeUTF(event.getClass().getName());
			event.write(serializer);
			return serializer.wrapAsByteBuffer();
		}
		catch (IOException e) {
			throw new IOException("Error while serializing event.", e);
		}
	}
}
 

private void sendEndOfSuperstepToAllIterationOutputs() throws IOException, InterruptedException {
	if (log.isDebugEnabled()) {
		log.debug(formatLogString("Sending end-of-superstep to all iteration outputs."));
	}

	for (RecordWriter<?> eventualOutput : this.eventualOutputs) {
		eventualOutput.broadcastEvent(EndOfSuperstepEvent.INSTANCE);
	}
}
 

/**
 * Tests {@link EventSerializer#isEvent(Buffer, Class)} returns
 * the correct answer for various encoded event buffers.
 */
@Test
public void testIsEvent() throws Exception {
	AbstractEvent[] events = {
		EndOfPartitionEvent.INSTANCE,
		EndOfSuperstepEvent.INSTANCE,
		new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forCheckpointWithDefaultLocation()),
		new TestTaskEvent(Math.random(), 12361231273L),
		new CancelCheckpointMarker(287087987329842L)
	};

	Class[] expectedClasses = Arrays.stream(events)
		.map(AbstractEvent::getClass)
		.toArray(Class[]::new);

	for (AbstractEvent evt : events) {
		for (Class<?> expectedClass: expectedClasses) {
			if (expectedClass.equals(TestTaskEvent.class)) {
				try {
					checkIsEvent(evt, expectedClass);
					fail("This should fail");
				}
				catch (UnsupportedOperationException ex) {
					// expected
				}
			}
			else if (evt.getClass().equals(expectedClass)) {
				assertTrue(checkIsEvent(evt, expectedClass));
			} else {
				assertFalse(checkIsEvent(evt, expectedClass));
			}
		}
	}
}
 

/**
 * Handles the event and returns whether the reader reached an end-of-stream event (either the
 * end of the whole stream or the end of an superstep).
 */
protected boolean handleEvent(AbstractEvent event) throws IOException {
	final Class<?> eventType = event.getClass();

	try {
		// ------------------------------------------------------------
		// Runtime events
		// ------------------------------------------------------------

		// This event is also checked at the (single) input gate to release the respective
		// channel, at which it was received.
		if (eventType == EndOfPartitionEvent.class) {
			return true;
		}
		else if (eventType == EndOfSuperstepEvent.class) {
			return incrementEndOfSuperstepEventAndCheck();
		}

		// ------------------------------------------------------------
		// Task events (user)
		// ------------------------------------------------------------
		else if (event instanceof TaskEvent) {
			taskEventHandler.publish((TaskEvent) event);

			return false;
		}
		else {
			throw new IllegalStateException("Received unexpected event of type " + eventType + " at reader.");
		}
	}
	catch (Throwable t) {
		throw new IOException("Error while handling event of type " + eventType + ": " + t.getMessage(), t);
	}
}
 

public static ByteBuffer toSerializedEvent(AbstractEvent event) throws IOException {
	final Class<?> eventClass = event.getClass();
	if (eventClass == EndOfPartitionEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_PARTITION_EVENT });
	}
	else if (eventClass == CheckpointBarrier.class) {
		return serializeCheckpointBarrier((CheckpointBarrier) event);
	}
	else if (eventClass == EndOfSuperstepEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_SUPERSTEP_EVENT });
	}
	else if (eventClass == CancelCheckpointMarker.class) {
		CancelCheckpointMarker marker = (CancelCheckpointMarker) event;

		ByteBuffer buf = ByteBuffer.allocate(12);
		buf.putInt(0, CANCEL_CHECKPOINT_MARKER_EVENT);
		buf.putLong(4, marker.getCheckpointId());
		return buf;
	}
	else {
		try {
			final DataOutputSerializer serializer = new DataOutputSerializer(128);
			serializer.writeInt(OTHER_EVENT);
			serializer.writeUTF(event.getClass().getName());
			event.write(serializer);
			return serializer.wrapAsByteBuffer();
		}
		catch (IOException e) {
			throw new IOException("Error while serializing event.", e);
		}
	}
}
 

private void sendEndOfSuperstepToAllIterationOutputs() throws IOException, InterruptedException {
	if (log.isDebugEnabled()) {
		log.debug(formatLogString("Sending end-of-superstep to all iteration outputs."));
	}

	for (RecordWriter<?> eventualOutput : this.eventualOutputs) {
		eventualOutput.broadcastEvent(EndOfSuperstepEvent.INSTANCE);
	}
}
 

/**
 * Tests {@link EventSerializer#isEvent(Buffer, Class)} returns
 * the correct answer for various encoded event buffers.
 */
@Test
public void testIsEvent() throws Exception {
	AbstractEvent[] events = {
		EndOfPartitionEvent.INSTANCE,
		EndOfSuperstepEvent.INSTANCE,
		new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forCheckpointWithDefaultLocation()),
		new TestTaskEvent(Math.random(), 12361231273L),
		new CancelCheckpointMarker(287087987329842L)
	};

	Class[] expectedClasses = Arrays.stream(events)
		.map(AbstractEvent::getClass)
		.toArray(Class[]::new);

	for (AbstractEvent evt : events) {
		for (Class<?> expectedClass: expectedClasses) {
			if (expectedClass.equals(TestTaskEvent.class)) {
				try {
					checkIsEvent(evt, expectedClass);
					fail("This should fail");
				}
				catch (UnsupportedOperationException ex) {
					// expected
				}
			}
			else if (evt.getClass().equals(expectedClass)) {
				assertTrue(checkIsEvent(evt, expectedClass));
			} else {
				assertFalse(checkIsEvent(evt, expectedClass));
			}
		}
	}
}
 

/**
 * Handles the event and returns whether the reader reached an end-of-stream event (either the
 * end of the whole stream or the end of an superstep).
 */
protected boolean handleEvent(AbstractEvent event) throws IOException {
	final Class<?> eventType = event.getClass();

	try {
		// ------------------------------------------------------------
		// Runtime events
		// ------------------------------------------------------------

		// This event is also checked at the (single) input gate to release the respective
		// channel, at which it was received.
		if (eventType == EndOfPartitionEvent.class) {
			return true;
		}
		else if (eventType == EndOfSuperstepEvent.class) {
			return incrementEndOfSuperstepEventAndCheck();
		}

		// ------------------------------------------------------------
		// Task events (user)
		// ------------------------------------------------------------
		else if (event instanceof TaskEvent) {
			taskEventHandler.publish((TaskEvent) event);

			return false;
		}
		else {
			throw new IllegalStateException("Received unexpected event of type " + eventType + " at reader.");
		}
	}
	catch (Throwable t) {
		throw new IOException("Error while handling event of type " + eventType + ": " + t.getMessage(), t);
	}
}
 

/**
 * Identifies whether the given buffer encodes the given event. Custom events are not supported.
 *
 * <p><strong>Pre-condition</strong>: This buffer must encode some event!</p>
 *
 * @param buffer the buffer to peak into
 * @param eventClass the expected class of the event type
 * @return whether the event class of the <tt>buffer</tt> matches the given <tt>eventClass</tt>
 */
private static boolean isEvent(ByteBuffer buffer, Class<?> eventClass) throws IOException {
	if (buffer.remaining() < 4) {
		throw new IOException("Incomplete event");
	}

	final int bufferPos = buffer.position();
	final ByteOrder bufferOrder = buffer.order();
	buffer.order(ByteOrder.BIG_ENDIAN);

	try {
		int type = buffer.getInt();

		if (eventClass.equals(EndOfPartitionEvent.class)) {
			return type == END_OF_PARTITION_EVENT;
		} else if (eventClass.equals(CheckpointBarrier.class)) {
			return type == CHECKPOINT_BARRIER_EVENT;
		} else if (eventClass.equals(EndOfSuperstepEvent.class)) {
			return type == END_OF_SUPERSTEP_EVENT;
		} else if (eventClass.equals(CancelCheckpointMarker.class)) {
			return type == CANCEL_CHECKPOINT_MARKER_EVENT;
		} else {
			throw new UnsupportedOperationException("Unsupported eventClass = " + eventClass);
		}
	}
	finally {
		buffer.order(bufferOrder);
		// restore the original position in the buffer (recall: we only peak into it!)
		buffer.position(bufferPos);
	}
}
 

public static ByteBuffer toSerializedEvent(AbstractEvent event) throws IOException {
	final Class<?> eventClass = event.getClass();
	if (eventClass == EndOfPartitionEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_PARTITION_EVENT });
	}
	else if (eventClass == CheckpointBarrier.class) {
		return serializeCheckpointBarrier((CheckpointBarrier) event);
	}
	else if (eventClass == EndOfSuperstepEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_SUPERSTEP_EVENT });
	}
	else if (eventClass == CancelCheckpointMarker.class) {
		CancelCheckpointMarker marker = (CancelCheckpointMarker) event;

		ByteBuffer buf = ByteBuffer.allocate(12);
		buf.putInt(0, CANCEL_CHECKPOINT_MARKER_EVENT);
		buf.putLong(4, marker.getCheckpointId());
		return buf;
	}
	else {
		try {
			final DataOutputSerializer serializer = new DataOutputSerializer(128);
			serializer.writeInt(OTHER_EVENT);
			serializer.writeUTF(event.getClass().getName());
			event.write(serializer);
			return serializer.wrapAsByteBuffer();
		}
		catch (IOException e) {
			throw new IOException("Error while serializing event.", e);
		}
	}
}
 

private void sendEndOfSuperstepToAllIterationOutputs() throws IOException, InterruptedException {
	if (log.isDebugEnabled()) {
		log.debug(formatLogString("Sending end-of-superstep to all iteration outputs."));
	}

	for (RecordWriter<?> eventualOutput : this.eventualOutputs) {
		eventualOutput.broadcastEvent(EndOfSuperstepEvent.INSTANCE);
	}
}
 

private void sendEndOfSuperstep() throws IOException, InterruptedException {
	for (RecordWriter eventualOutput : this.eventualOutputs) {
		eventualOutput.broadcastEvent(EndOfSuperstepEvent.INSTANCE);
	}
}
 

private void sendEndOfSuperstep() throws IOException, InterruptedException {
	for (RecordWriter eventualOutput : this.eventualOutputs) {
		eventualOutput.broadcastEvent(EndOfSuperstepEvent.INSTANCE);
	}
}
 

private void sendEndOfSuperstep() throws IOException, InterruptedException {
	for (RecordWriter eventualOutput : this.eventualOutputs) {
		eventualOutput.broadcastEvent(EndOfSuperstepEvent.INSTANCE);
	}
}