org.apache.flink.runtime.event.TaskEvent Java Examples

/** Barrier will release the waiting thread if an event occurs. */
@Override
public void onEvent(TaskEvent event) {
	if (event instanceof TerminationEvent) {
		terminationSignaled = true;
	}
	else if (event instanceof AllWorkersDoneEvent) {
		AllWorkersDoneEvent wde = (AllWorkersDoneEvent) event;
		aggregatorNames = wde.getAggregatorNames();
		aggregates = wde.getAggregates(userCodeClassLoader);
	}
	else {
		throw new IllegalArgumentException("Unknown event type.");
	}

	latch.countDown();
}
 

@Test
@SuppressWarnings("unchecked")
public void testTaskEvent() throws Exception {
	final AbstractReader reader = new MockReader(createInputGate(1));

	final EventListener<TaskEvent> listener1 = mock(EventListener.class);
	final EventListener<TaskEvent> listener2 = mock(EventListener.class);
	final EventListener<TaskEvent> listener3 = mock(EventListener.class);

	reader.registerTaskEventListener(listener1, TestTaskEvent1.class);
	reader.registerTaskEventListener(listener2, TestTaskEvent2.class);
	reader.registerTaskEventListener(listener3, TaskEvent.class);

	reader.handleEvent(new TestTaskEvent1()); // for listener1 only
	reader.handleEvent(new TestTaskEvent2()); // for listener2 only

	verify(listener1, times(1)).onEvent(Matchers.any(TaskEvent.class));
	verify(listener2, times(1)).onEvent(Matchers.any(TaskEvent.class));
	verify(listener3, times(0)).onEvent(Matchers.any(TaskEvent.class));
}
 

/**
 * Sends a task event backwards to an intermediate result partition producer.
 * <p>
 * Backwards task events flow between readers and writers and therefore
 * will only work when both are running at the same time, which is only
 * guaranteed to be the case when both the respective producer and
 * consumer task run pipelined.
 */
public void sendTaskEvent(ResultPartitionID partitionId, TaskEvent event, final RemoteInputChannel inputChannel) throws IOException {
	checkNotClosed();

	tcpChannel.writeAndFlush(new TaskEventRequest(event, partitionId, inputChannel.getInputChannelId()))
			.addListener(
					new ChannelFutureListener() {
						@Override
						public void operationComplete(ChannelFuture future) throws Exception {
							if (!future.isSuccess()) {
								SocketAddress remoteAddr = future.channel().remoteAddress();
								inputChannel.onError(new LocalTransportException(
									String.format("Sending the task event to '%s' failed.", remoteAddr),
									future.channel().localAddress(), future.cause()
								));
							}
						}
					});
}
 

/** Barrier will release the waiting thread if an event occurs. */
@Override
public void onEvent(TaskEvent event) {
	if (event instanceof TerminationEvent) {
		terminationSignaled = true;
	}
	else if (event instanceof AllWorkersDoneEvent) {
		AllWorkersDoneEvent wde = (AllWorkersDoneEvent) event;
		aggregatorNames = wde.getAggregatorNames();
		aggregates = wde.getAggregates(userCodeClassLoader);
	}
	else {
		throw new IllegalArgumentException("Unknown event type.");
	}

	latch.countDown();
}
 

static TaskEventRequest readFrom(ByteBuf buffer, ClassLoader classLoader) throws IOException {
	// directly deserialize fromNetty's buffer
	int length = buffer.readInt();
	ByteBuffer serializedEvent = buffer.nioBuffer(buffer.readerIndex(), length);
	// assume this event's content is read from the ByteBuf (positions are not shared!)
	buffer.readerIndex(buffer.readerIndex() + length);

	TaskEvent event =
		(TaskEvent) EventSerializer.fromSerializedEvent(serializedEvent, classLoader);

	ResultPartitionID partitionId =
		new ResultPartitionID(
			IntermediateResultPartitionID.fromByteBuf(buffer),
			ExecutionAttemptID.fromByteBuf(buffer));

	InputChannelID receiverId = InputChannelID.fromByteBuf(buffer);

	return new TaskEventRequest(event, partitionId, receiverId);
}
 

private void sync(TaskEvent event) throws InterruptedException {

		TerminationSignaled terminationSignaled = new TerminationSignaled();

		SuperstepBarrier barrier = new SuperstepBarrier(getClass().getClassLoader());
		barrier.setup();

		Thread headThread = new Thread(new IterationHead(barrier, terminationSignaled));
		Thread syncThread = new Thread(new IterationSync(barrier, event));

		headThread.start();
		syncThread.start();

		headThread.join();
		syncThread.join();

		if (event instanceof TerminationEvent) {
			assertTrue(terminationSignaled.isTerminationSignaled());
		} else {
			assertFalse(terminationSignaled.isTerminationSignaled());
		}
	}
 

/**
 * Publishes the event to the registered {@link EventListener} instances.
 *
 * <p>This method is either called directly from a {@link LocalInputChannel} or the network I/O
 * thread on behalf of a {@link RemoteInputChannel}.
 *
 * @return whether the event was published to a registered event handler (initiated via {@link
 * #registerPartition(ResultPartitionID)}) or not
 */
@Override
public boolean publish(ResultPartitionID partitionId, TaskEvent event) {
	checkNotNull(partitionId);
	checkNotNull(event);

	TaskEventHandler taskEventHandler;
	synchronized (registeredHandlers) {
		taskEventHandler = registeredHandlers.get(partitionId);
	}

	if (taskEventHandler != null) {
		taskEventHandler.publish(event);
		return true;
	}

	return false;
}
 

/**
 * Subscribes a listener to this dispatcher for events on a partition.
 *
 * @param partitionId
 * 		ID of the partition to subscribe for (must be registered via {@link
 * 		#registerPartition(ResultPartitionID)} first!)
 * @param eventListener
 * 		the event listener to subscribe
 * @param eventType
 * 		event type to subscribe to
 */
public void subscribeToEvent(
		ResultPartitionID partitionId,
		EventListener<TaskEvent> eventListener,
		Class<? extends TaskEvent> eventType) {
	checkNotNull(partitionId);
	checkNotNull(eventListener);
	checkNotNull(eventType);

	TaskEventHandler taskEventHandler;
	synchronized (registeredHandlers) {
		taskEventHandler = registeredHandlers.get(partitionId);
	}
	if (taskEventHandler == null) {
		throw new IllegalStateException(
			"Partition " + partitionId + " not registered at task event dispatcher.");
	}
	taskEventHandler.subscribe(eventListener, eventType);
}
 

/**
 * Publishes the event to the registered {@link EventListener} instances.
 *
 * <p>This method is either called directly from a {@link LocalInputChannel} or the network I/O
 * thread on behalf of a {@link RemoteInputChannel}.
 *
 * @return whether the event was published to a registered event handler (initiated via {@link
 * #registerPartition(ResultPartitionID)}) or not
 */
public boolean publish(ResultPartitionID partitionId, TaskEvent event) {
	checkNotNull(partitionId);
	checkNotNull(event);

	TaskEventHandler taskEventHandler;
	synchronized (registeredHandlers) {
		taskEventHandler = registeredHandlers.get(partitionId);
	}

	if (taskEventHandler != null) {
		taskEventHandler.publish(event);
		return true;
	}

	return false;
}
 

private void sync(TaskEvent event) throws InterruptedException {

		TerminationSignaled terminationSignaled = new TerminationSignaled();

		SuperstepBarrier barrier = new SuperstepBarrier(getClass().getClassLoader());
		barrier.setup();

		Thread headThread = new Thread(new IterationHead(barrier, terminationSignaled));
		Thread syncThread = new Thread(new IterationSync(barrier, event));

		headThread.start();
		syncThread.start();

		headThread.join();
		syncThread.join();

		if (event instanceof TerminationEvent) {
			assertTrue(terminationSignaled.isTerminationSignaled());
		} else {
			assertFalse(terminationSignaled.isTerminationSignaled());
		}
	}
 

/**
 * Subscribes a listener to this dispatcher for events on a partition.
 *
 * @param partitionId
 * 		ID of the partition to subscribe for (must be registered via {@link
 * 		#registerPartition(ResultPartitionID)} first!)
 * @param eventListener
 * 		the event listener to subscribe
 * @param eventType
 * 		event type to subscribe to
 */
public void subscribeToEvent(
		ResultPartitionID partitionId,
		EventListener<TaskEvent> eventListener,
		Class<? extends TaskEvent> eventType) {
	checkNotNull(partitionId);
	checkNotNull(eventListener);
	checkNotNull(eventType);

	TaskEventHandler taskEventHandler;
	synchronized (registeredHandlers) {
		taskEventHandler = registeredHandlers.get(partitionId);
	}
	if (taskEventHandler == null) {
		throw new IllegalStateException(
			"Partition " + partitionId + " not registered at task event dispatcher.");
	}
	taskEventHandler.subscribe(eventListener, eventType);
}
 

@Test
@SuppressWarnings("unchecked")
public void testTaskEvent() throws Exception {
	final AbstractReader reader = new MockReader(createInputGate(1));

	final EventListener<TaskEvent> listener1 = mock(EventListener.class);
	final EventListener<TaskEvent> listener2 = mock(EventListener.class);
	final EventListener<TaskEvent> listener3 = mock(EventListener.class);

	reader.registerTaskEventListener(listener1, TestTaskEvent1.class);
	reader.registerTaskEventListener(listener2, TestTaskEvent2.class);
	reader.registerTaskEventListener(listener3, TaskEvent.class);

	reader.handleEvent(new TestTaskEvent1()); // for listener1 only
	reader.handleEvent(new TestTaskEvent2()); // for listener2 only

	verify(listener1, times(1)).onEvent(Matchers.any(TaskEvent.class));
	verify(listener2, times(1)).onEvent(Matchers.any(TaskEvent.class));
	verify(listener3, times(0)).onEvent(Matchers.any(TaskEvent.class));
}
 

static TaskEventRequest readFrom(ByteBuf buffer, ClassLoader classLoader) throws IOException {
	// directly deserialize fromNetty's buffer
	int length = buffer.readInt();
	ByteBuffer serializedEvent = buffer.nioBuffer(buffer.readerIndex(), length);
	// assume this event's content is read from the ByteBuf (positions are not shared!)
	buffer.readerIndex(buffer.readerIndex() + length);

	TaskEvent event =
		(TaskEvent) EventSerializer.fromSerializedEvent(serializedEvent, classLoader);

	ResultPartitionID partitionId =
		new ResultPartitionID(
			IntermediateResultPartitionID.fromByteBuf(buffer),
			ExecutionAttemptID.fromByteBuf(buffer));

	InputChannelID receiverId = InputChannelID.fromByteBuf(buffer);

	return new TaskEventRequest(event, partitionId, receiverId);
}
 

/**
 * Sends a task event backwards to an intermediate result partition producer.
 *
 * <p>Backwards task events flow between readers and writers and therefore
 * will only work when both are running at the same time, which is only
 * guaranteed to be the case when both the respective producer and
 * consumer task run pipelined.
 */
@Override
public void sendTaskEvent(ResultPartitionID partitionId, TaskEvent event, final RemoteInputChannel inputChannel) throws IOException {
	checkNotClosed();

	tcpChannel.writeAndFlush(new TaskEventRequest(event, partitionId, inputChannel.getInputChannelId()))
			.addListener(
					new ChannelFutureListener() {
						@Override
						public void operationComplete(ChannelFuture future) throws Exception {
							if (!future.isSuccess()) {
								SocketAddress remoteAddr = future.channel().remoteAddress();
								inputChannel.onError(new LocalTransportException(
									String.format("Sending the task event to '%s' failed.", remoteAddr),
									future.channel().localAddress(), future.cause()
								));
							}
						}
					});
}
 

@Test
public void subscribeToEventNotRegistered() throws Exception {
	TaskEventDispatcher ted = new TaskEventDispatcher();

	expectedException.expect(IllegalStateException.class);
	expectedException.expectMessage("not registered at task event dispatcher");

	ted.subscribeToEvent(new ResultPartitionID(), new ZeroShotEventListener(), TaskEvent.class);
}
 

@Override
public void sendTaskEvent(TaskEvent event) throws IOException {
	synchronized (requestLock) {
		for (InputChannel inputChannel : inputChannels.values()) {
			inputChannel.sendTaskEvent(event);
		}

		if (numberOfUninitializedChannels > 0) {
			pendingEvents.add(event);
		}
	}
}
 

@Override
void sendTaskEvent(TaskEvent event) throws IOException {
	checkState(!isReleased.get(), "Tried to send task event to producer after channel has been released.");
	checkState(partitionRequestClient != null, "Tried to send task event to producer before requesting a queue.");

	checkError();

	partitionRequestClient.sendTaskEvent(partitionId, event, this);
}
 

@Override
void sendTaskEvent(TaskEvent event) throws IOException {
	checkError();
	checkState(subpartitionView != null, "Tried to send task event to producer before requesting the subpartition.");

	if (!taskEventDispatcher.publish(partitionId, event)) {
		throw new IOException("Error while publishing event " + event + " to producer. The producer could not be found.");
	}
}
 

/**
 * Handles the event and returns whether the reader reached an end-of-stream event (either the
 * end of the whole stream or the end of an superstep).
 */
protected boolean handleEvent(AbstractEvent event) throws IOException {
	final Class<?> eventType = event.getClass();

	try {
		// ------------------------------------------------------------
		// Runtime events
		// ------------------------------------------------------------

		// This event is also checked at the (single) input gate to release the respective
		// channel, at which it was received.
		if (eventType == EndOfPartitionEvent.class) {
			return true;
		}
		else if (eventType == EndOfSuperstepEvent.class) {
			return incrementEndOfSuperstepEventAndCheck();
		}

		// ------------------------------------------------------------
		// Task events (user)
		// ------------------------------------------------------------
		else if (event instanceof TaskEvent) {
			taskEventHandler.publish((TaskEvent) event);

			return false;
		}
		else {
			throw new IllegalStateException("Received unexpected event of type " + eventType + " at reader.");
		}
	}
	catch (Throwable t) {
		throw new IOException("Error while handling event of type " + eventType + ": " + t.getMessage(), t);
	}
}
 

@Override
public void sendTaskEvent(TaskEvent event) throws IOException {
	synchronized (requestLock) {
		for (InputChannel inputChannel : inputChannels.values()) {
			inputChannel.sendTaskEvent(event);
		}

		if (numberOfUninitializedChannels > 0) {
			pendingEvents.add(event);
		}
	}
}
 

@Test
public void subscribeToEventNotRegistered() throws Exception {
	TaskEventDispatcher ted = new TaskEventDispatcher();

	expectedException.expect(IllegalStateException.class);
	expectedException.expectMessage("not registered at task event dispatcher");

	ted.subscribeToEvent(new ResultPartitionID(), new ZeroShotEventListener(), TaskEvent.class);
}
 

/**
 * Tests {@link TaskEventDispatcher#publish(ResultPartitionID, TaskEvent)} and {@link TaskEventDispatcher#subscribeToEvent(ResultPartitionID, EventListener, Class)} methods.
 */
@Test
public void publishSubscribe() throws Exception {
	ResultPartitionID partitionId1 = new ResultPartitionID();
	ResultPartitionID partitionId2 = new ResultPartitionID();
	TaskEventDispatcher ted = new TaskEventDispatcher();

	AllWorkersDoneEvent event1 = new AllWorkersDoneEvent();
	TerminationEvent event2 = new TerminationEvent();
	assertFalse(ted.publish(partitionId1, event1));

	ted.registerPartition(partitionId1);
	ted.registerPartition(partitionId2);

	// no event listener subscribed yet, but the event is forwarded to a TaskEventHandler
	assertTrue(ted.publish(partitionId1, event1));

	OneShotEventListener eventListener1a = new OneShotEventListener(event1);
	ZeroShotEventListener eventListener1b = new ZeroShotEventListener();
	ZeroShotEventListener eventListener2 = new ZeroShotEventListener();
	OneShotEventListener eventListener3 = new OneShotEventListener(event2);
	ted.subscribeToEvent(partitionId1, eventListener1a, AllWorkersDoneEvent.class);
	ted.subscribeToEvent(partitionId2, eventListener1b, AllWorkersDoneEvent.class);
	ted.subscribeToEvent(partitionId1, eventListener2, TaskEvent.class);
	ted.subscribeToEvent(partitionId1, eventListener3, TerminationEvent.class);

	assertTrue(ted.publish(partitionId1, event1));
	assertTrue("listener should have fired for AllWorkersDoneEvent", eventListener1a.fired);
	assertFalse("listener should not have fired for AllWorkersDoneEvent", eventListener3.fired);

	// publish another event, verify that only the right subscriber is called
	assertTrue(ted.publish(partitionId1, event2));
	assertTrue("listener should have fired for TerminationEvent", eventListener3.fired);
}
 

@Override
void sendTaskEvent(TaskEvent event) throws IOException {
	checkState(!isReleased.get(), "Tried to send task event to producer after channel has been released.");
	checkState(partitionRequestClient != null, "Tried to send task event to producer before requesting a queue.");

	checkError();

	partitionRequestClient.sendTaskEvent(partitionId, event, this);
}
 

/**
 * Publishes the task event to all subscribed event listeners.
 *
 * @param event The event to publish.
 */
public void publish(TaskEvent event) {
	synchronized (listeners) {
		for (EventListener<TaskEvent> listener : listeners.get(event.getClass())) {
			listener.onEvent(event);
		}
	}
}
 

@Override
void sendTaskEvent(TaskEvent event) throws IOException {
	checkError();
	checkState(subpartitionView != null, "Tried to send task event to producer before requesting the subpartition.");

	if (!taskEventPublisher.publish(partitionId, event)) {
		throw new IOException("Error while publishing event " + event + " to producer. The producer could not be found.");
	}
}
 

@Override
public void onEvent(TaskEvent event) {
	if (WorkerDoneEvent.class.equals(event.getClass())) {
		onWorkerDoneEvent((WorkerDoneEvent) event);
		return;
	}
	throw new IllegalStateException("Unable to handle event " + event.getClass().getName());
}
 

/**
 * Handles the event and returns whether the reader reached an end-of-stream event (either the
 * end of the whole stream or the end of an superstep).
 */
protected boolean handleEvent(AbstractEvent event) throws IOException {
	final Class<?> eventType = event.getClass();

	try {
		// ------------------------------------------------------------
		// Runtime events
		// ------------------------------------------------------------

		// This event is also checked at the (single) input gate to release the respective
		// channel, at which it was received.
		if (eventType == EndOfPartitionEvent.class) {
			return true;
		}
		else if (eventType == EndOfSuperstepEvent.class) {
			return incrementEndOfSuperstepEventAndCheck();
		}

		// ------------------------------------------------------------
		// Task events (user)
		// ------------------------------------------------------------
		else if (event instanceof TaskEvent) {
			taskEventHandler.publish((TaskEvent) event);

			return false;
		}
		else {
			throw new IllegalStateException("Received unexpected event of type " + eventType + " at reader.");
		}
	}
	catch (Throwable t) {
		throw new IOException("Error while handling event of type " + eventType + ": " + t.getMessage(), t);
	}
}
 

@Override
public void onEvent(TaskEvent event) {
	if (WorkerDoneEvent.class.equals(event.getClass())) {
		onWorkerDoneEvent((WorkerDoneEvent) event);
		return;
	}
	throw new IllegalStateException("Unable to handle event " + event.getClass().getName());
}
 

/**
 * Tests {@link TaskEventDispatcher#publish(ResultPartitionID, TaskEvent)} and {@link TaskEventDispatcher#subscribeToEvent(ResultPartitionID, EventListener, Class)} methods.
 */
@Test
public void publishSubscribe() throws Exception {
	ResultPartitionID partitionId1 = new ResultPartitionID();
	ResultPartitionID partitionId2 = new ResultPartitionID();
	TaskEventDispatcher ted = new TaskEventDispatcher();

	AllWorkersDoneEvent event1 = new AllWorkersDoneEvent();
	TerminationEvent event2 = new TerminationEvent();
	assertFalse(ted.publish(partitionId1, event1));

	ted.registerPartition(partitionId1);
	ted.registerPartition(partitionId2);

	// no event listener subscribed yet, but the event is forwarded to a TaskEventHandler
	assertTrue(ted.publish(partitionId1, event1));

	OneShotEventListener eventListener1a = new OneShotEventListener(event1);
	ZeroShotEventListener eventListener1b = new ZeroShotEventListener();
	ZeroShotEventListener eventListener2 = new ZeroShotEventListener();
	OneShotEventListener eventListener3 = new OneShotEventListener(event2);
	ted.subscribeToEvent(partitionId1, eventListener1a, AllWorkersDoneEvent.class);
	ted.subscribeToEvent(partitionId2, eventListener1b, AllWorkersDoneEvent.class);
	ted.subscribeToEvent(partitionId1, eventListener2, TaskEvent.class);
	ted.subscribeToEvent(partitionId1, eventListener3, TerminationEvent.class);

	assertTrue(ted.publish(partitionId1, event1));
	assertTrue("listener should have fired for AllWorkersDoneEvent", eventListener1a.fired);
	assertFalse("listener should not have fired for AllWorkersDoneEvent", eventListener3.fired);

	// publish another event, verify that only the right subscriber is called
	assertTrue(ted.publish(partitionId1, event2));
	assertTrue("listener should have fired for TerminationEvent", eventListener3.fired);
}
 

/**
 * Publishes the task event to all subscribed event listeners.
 *
 * @param event The event to publish.
 */
public void publish(TaskEvent event) {
	synchronized (listeners) {
		for (EventListener<TaskEvent> listener : listeners.get(event.getClass())) {
			listener.onEvent(event);
		}
	}
}