org.apache.flink.runtime.io.network.util.TestTaskEvent Java Examples

@Test
public void testSerializeDeserializeEvent() throws Exception {
	AbstractEvent[] events = {
			EndOfPartitionEvent.INSTANCE,
			EndOfSuperstepEvent.INSTANCE,
			new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forCheckpointWithDefaultLocation()),
			new TestTaskEvent(Math.random(), 12361231273L),
			new CancelCheckpointMarker(287087987329842L)
	};

	for (AbstractEvent evt : events) {
		ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
		assertTrue(serializedEvent.hasRemaining());

		AbstractEvent deserialized =
				EventSerializer.fromSerializedEvent(serializedEvent, getClass().getClassLoader());
		assertNotNull(deserialized);
		assertEquals(evt, deserialized);
	}
}
 

@Test
public void testSerializeDeserializeEvent() throws Exception {
	AbstractEvent[] events = {
			EndOfPartitionEvent.INSTANCE,
			EndOfSuperstepEvent.INSTANCE,
			new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forCheckpointWithDefaultLocation()),
			new TestTaskEvent(Math.random(), 12361231273L),
			new CancelCheckpointMarker(287087987329842L)
	};

	for (AbstractEvent evt : events) {
		ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
		assertTrue(serializedEvent.hasRemaining());

		AbstractEvent deserialized =
				EventSerializer.fromSerializedEvent(serializedEvent, getClass().getClassLoader());
		assertNotNull(deserialized);
		assertEquals(evt, deserialized);
	}
}
 

/**
 * Tests {@link EventSerializer#isEvent(Buffer, Class)} returns
 * the correct answer for various encoded event buffers.
 */
@Test
public void testIsEvent() throws Exception {
	Class[] expectedClasses = Arrays.stream(events)
		.map(AbstractEvent::getClass)
		.toArray(Class[]::new);

	for (AbstractEvent evt : events) {
		for (Class<?> expectedClass: expectedClasses) {
			if (expectedClass.equals(TestTaskEvent.class)) {
				try {
					checkIsEvent(evt, expectedClass);
					fail("This should fail");
				}
				catch (UnsupportedOperationException ex) {
					// expected
				}
			}
			else if (evt.getClass().equals(expectedClass)) {
				assertTrue(checkIsEvent(evt, expectedClass));
			} else {
				assertFalse(checkIsEvent(evt, expectedClass));
			}
		}
	}
}
 

/**
 * Tests {@link EventSerializer#isEvent(Buffer, Class)} returns
 * the correct answer for various encoded event buffers.
 */
@Test
public void testIsEvent() throws Exception {
	AbstractEvent[] events = {
		EndOfPartitionEvent.INSTANCE,
		EndOfSuperstepEvent.INSTANCE,
		new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forCheckpointWithDefaultLocation()),
		new TestTaskEvent(Math.random(), 12361231273L),
		new CancelCheckpointMarker(287087987329842L)
	};

	Class[] expectedClasses = Arrays.stream(events)
		.map(AbstractEvent::getClass)
		.toArray(Class[]::new);

	for (AbstractEvent evt : events) {
		for (Class<?> expectedClass: expectedClasses) {
			if (expectedClass.equals(TestTaskEvent.class)) {
				try {
					checkIsEvent(evt, expectedClass);
					fail("This should fail");
				}
				catch (UnsupportedOperationException ex) {
					// expected
				}
			}
			else if (evt.getClass().equals(expectedClass)) {
				assertTrue(checkIsEvent(evt, expectedClass));
			} else {
				assertFalse(checkIsEvent(evt, expectedClass));
			}
		}
	}
}
 

/**
 * Tests {@link EventSerializer#isEvent(Buffer, Class)} returns
 * the correct answer for various encoded event buffers.
 */
@Test
public void testIsEvent() throws Exception {
	AbstractEvent[] events = {
		EndOfPartitionEvent.INSTANCE,
		EndOfSuperstepEvent.INSTANCE,
		new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forCheckpointWithDefaultLocation()),
		new TestTaskEvent(Math.random(), 12361231273L),
		new CancelCheckpointMarker(287087987329842L)
	};

	Class[] expectedClasses = Arrays.stream(events)
		.map(AbstractEvent::getClass)
		.toArray(Class[]::new);

	for (AbstractEvent evt : events) {
		for (Class<?> expectedClass: expectedClasses) {
			if (expectedClass.equals(TestTaskEvent.class)) {
				try {
					checkIsEvent(evt, expectedClass);
					fail("This should fail");
				}
				catch (UnsupportedOperationException ex) {
					// expected
				}
			}
			else if (evt.getClass().equals(expectedClass)) {
				assertTrue(checkIsEvent(evt, expectedClass));
			} else {
				assertFalse(checkIsEvent(evt, expectedClass));
			}
		}
	}
}
 

@Test
public void testBackwardsEventWithUninitializedChannel() throws Exception {
	// Setup environment
	final TaskEventDispatcher taskEventDispatcher = mock(TaskEventDispatcher.class);
	when(taskEventDispatcher.publish(any(ResultPartitionID.class), any(TaskEvent.class))).thenReturn(true);

	final ResultSubpartitionView iterator = mock(ResultSubpartitionView.class);
	when(iterator.getNextBuffer()).thenReturn(
		new BufferAndBacklog(new NetworkBuffer(MemorySegmentFactory.allocateUnpooledSegment(1024), FreeingBufferRecycler.INSTANCE), false, 0, false));

	final ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
	when(partitionManager.createSubpartitionView(
		any(ResultPartitionID.class),
		anyInt(),
		any(BufferAvailabilityListener.class))).thenReturn(iterator);

	// Setup reader with one local and one unknown input channel

	final SingleInputGate inputGate = createInputGate();
	final BufferPool bufferPool = mock(BufferPool.class);
	when(bufferPool.getNumberOfRequiredMemorySegments()).thenReturn(2);

	inputGate.setBufferPool(bufferPool);

	// Local
	ResultPartitionID localPartitionId = new ResultPartitionID(new IntermediateResultPartitionID(), new ExecutionAttemptID());

	InputChannel local = new LocalInputChannel(inputGate, 0, localPartitionId, partitionManager, taskEventDispatcher, UnregisteredMetricGroups.createUnregisteredTaskMetricGroup().getIOMetricGroup());

	// Unknown
	ResultPartitionID unknownPartitionId = new ResultPartitionID(new IntermediateResultPartitionID(), new ExecutionAttemptID());

	InputChannel unknown = new UnknownInputChannel(inputGate, 1, unknownPartitionId, partitionManager, taskEventDispatcher, mock(ConnectionManager.class), 0, 0, UnregisteredMetricGroups.createUnregisteredTaskMetricGroup().getIOMetricGroup());

	// Set channels
	inputGate.setInputChannel(localPartitionId.getPartitionId(), local);
	inputGate.setInputChannel(unknownPartitionId.getPartitionId(), unknown);

	// Request partitions
	inputGate.requestPartitions();

	// Only the local channel can request
	verify(partitionManager, times(1)).createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferAvailabilityListener.class));

	// Send event backwards and initialize unknown channel afterwards
	final TaskEvent event = new TestTaskEvent();
	inputGate.sendTaskEvent(event);

	// Only the local channel can send out the event
	verify(taskEventDispatcher, times(1)).publish(any(ResultPartitionID.class), any(TaskEvent.class));

	// After the update, the pending event should be send to local channel
	inputGate.updateInputChannel(new InputChannelDeploymentDescriptor(new ResultPartitionID(unknownPartitionId.getPartitionId(), unknownPartitionId.getProducerId()), ResultPartitionLocation.createLocal()));

	verify(partitionManager, times(2)).createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferAvailabilityListener.class));
	verify(taskEventDispatcher, times(2)).publish(any(ResultPartitionID.class), any(TaskEvent.class));
}
 

@Test
public void testBackwardsEventWithUninitializedChannel() throws Exception {
	// Setup environment
	final TaskEventDispatcher taskEventDispatcher = mock(TaskEventDispatcher.class);
	when(taskEventDispatcher.publish(any(ResultPartitionID.class), any(TaskEvent.class))).thenReturn(true);

	final ResultSubpartitionView iterator = mock(ResultSubpartitionView.class);
	when(iterator.getNextBuffer()).thenReturn(
		new BufferAndBacklog(new NetworkBuffer(MemorySegmentFactory.allocateUnpooledSegment(1024), FreeingBufferRecycler.INSTANCE), false, 0, false));

	final ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
	when(partitionManager.createSubpartitionView(
		any(ResultPartitionID.class),
		anyInt(),
		any(BufferAvailabilityListener.class))).thenReturn(iterator);

	// Setup reader with one local and one unknown input channel

	NettyShuffleEnvironment environment = createNettyShuffleEnvironment();
	final SingleInputGate inputGate = createInputGate(environment, 2, ResultPartitionType.PIPELINED);
	try {
		// Local
		ResultPartitionID localPartitionId = new ResultPartitionID();

		InputChannelBuilder.newBuilder()
			.setPartitionId(localPartitionId)
			.setPartitionManager(partitionManager)
			.setTaskEventPublisher(taskEventDispatcher)
			.buildLocalAndSetToGate(inputGate);

		// Unknown
		ResultPartitionID unknownPartitionId = new ResultPartitionID();

		InputChannelBuilder.newBuilder()
			.setChannelIndex(1)
			.setPartitionId(unknownPartitionId)
			.setPartitionManager(partitionManager)
			.setTaskEventPublisher(taskEventDispatcher)
			.buildUnknownAndSetToGate(inputGate);

		inputGate.setup();

		// Only the local channel can request
		verify(partitionManager, times(1)).createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferAvailabilityListener.class));

		// Send event backwards and initialize unknown channel afterwards
		final TaskEvent event = new TestTaskEvent();
		inputGate.sendTaskEvent(event);

		// Only the local channel can send out the event
		verify(taskEventDispatcher, times(1)).publish(any(ResultPartitionID.class), any(TaskEvent.class));

		// After the update, the pending event should be send to local channel

		ResourceID location = ResourceID.generate();
		inputGate.updateInputChannel(location, createRemoteWithIdAndLocation(unknownPartitionId.getPartitionId(), location));

		verify(partitionManager, times(2)).createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferAvailabilityListener.class));
		verify(taskEventDispatcher, times(2)).publish(any(ResultPartitionID.class), any(TaskEvent.class));
	}
	finally {
		inputGate.close();
		environment.close();
	}
}
 

@Test
public void testBackwardsEventWithUninitializedChannel() throws Exception {
	// Setup environment
	TestingTaskEventPublisher taskEventPublisher = new TestingTaskEventPublisher();

	TestingResultPartitionManager partitionManager = new TestingResultPartitionManager(new NoOpResultSubpartitionView());

	// Setup reader with one local and one unknown input channel

	NettyShuffleEnvironment environment = createNettyShuffleEnvironment();
	final SingleInputGate inputGate = createInputGate(environment, 2, ResultPartitionType.PIPELINED);
	final InputChannel[] inputChannels = new InputChannel[2];
	try {
		// Local
		ResultPartitionID localPartitionId = new ResultPartitionID();

		inputChannels[0] = InputChannelBuilder.newBuilder()
			.setPartitionId(localPartitionId)
			.setPartitionManager(partitionManager)
			.setTaskEventPublisher(taskEventPublisher)
			.buildLocalChannel(inputGate);

		// Unknown
		ResultPartitionID unknownPartitionId = new ResultPartitionID();

		inputChannels[1] = InputChannelBuilder.newBuilder()
			.setChannelIndex(1)
			.setPartitionId(unknownPartitionId)
			.setPartitionManager(partitionManager)
			.setTaskEventPublisher(taskEventPublisher)
			.buildUnknownChannel(inputGate);

		setupInputGate(inputGate, inputChannels);

		// Only the local channel can request
		assertEquals(1, partitionManager.counter);

		// Send event backwards and initialize unknown channel afterwards
		final TaskEvent event = new TestTaskEvent();
		inputGate.sendTaskEvent(event);

		// Only the local channel can send out the event
		assertEquals(1, taskEventPublisher.counter);

		// After the update, the pending event should be send to local channel

		ResourceID location = ResourceID.generate();
		inputGate.updateInputChannel(location, createRemoteWithIdAndLocation(unknownPartitionId.getPartitionId(), location));

		assertEquals(2, partitionManager.counter);
		assertEquals(2, taskEventPublisher.counter);
	}
	finally {
		inputGate.close();
		environment.close();
	}
}