org.apache.flink.runtime.iterative.event.AllWorkersDoneEvent Java Examples

/** Barrier will release the waiting thread if an event occurs. */
@Override
public void onEvent(TaskEvent event) {
	if (event instanceof TerminationEvent) {
		terminationSignaled = true;
	}
	else if (event instanceof AllWorkersDoneEvent) {
		AllWorkersDoneEvent wde = (AllWorkersDoneEvent) event;
		aggregatorNames = wde.getAggregatorNames();
		aggregates = wde.getAggregates(userCodeClassLoader);
	}
	else {
		throw new IllegalArgumentException("Unknown event type.");
	}

	latch.countDown();
}
 

/** Barrier will release the waiting thread if an event occurs. */
@Override
public void onEvent(TaskEvent event) {
	if (event instanceof TerminationEvent) {
		terminationSignaled = true;
	}
	else if (event instanceof AllWorkersDoneEvent) {
		AllWorkersDoneEvent wde = (AllWorkersDoneEvent) event;
		aggregatorNames = wde.getAggregatorNames();
		aggregates = wde.getAggregates(userCodeClassLoader);
	}
	else {
		throw new IllegalArgumentException("Unknown event type.");
	}

	latch.countDown();
}
 

/** Barrier will release the waiting thread if an event occurs. */
@Override
public void onEvent(TaskEvent event) {
	if (event instanceof TerminationEvent) {
		terminationSignaled = true;
	}
	else if (event instanceof AllWorkersDoneEvent) {
		AllWorkersDoneEvent wde = (AllWorkersDoneEvent) event;
		aggregatorNames = wde.getAggregatorNames();
		aggregates = wde.getAggregates(userCodeClassLoader);
	}
	else {
		throw new IllegalArgumentException("Unknown event type.");
	}

	latch.countDown();
}
 

/**
 * Tests {@link TaskEventDispatcher#publish(ResultPartitionID, TaskEvent)} and {@link TaskEventDispatcher#subscribeToEvent(ResultPartitionID, EventListener, Class)} methods.
 */
@Test
public void publishSubscribe() throws Exception {
	ResultPartitionID partitionId1 = new ResultPartitionID();
	ResultPartitionID partitionId2 = new ResultPartitionID();
	TaskEventDispatcher ted = new TaskEventDispatcher();

	AllWorkersDoneEvent event1 = new AllWorkersDoneEvent();
	TerminationEvent event2 = new TerminationEvent();
	assertFalse(ted.publish(partitionId1, event1));

	ted.registerPartition(partitionId1);
	ted.registerPartition(partitionId2);

	// no event listener subscribed yet, but the event is forwarded to a TaskEventHandler
	assertTrue(ted.publish(partitionId1, event1));

	OneShotEventListener eventListener1a = new OneShotEventListener(event1);
	ZeroShotEventListener eventListener1b = new ZeroShotEventListener();
	ZeroShotEventListener eventListener2 = new ZeroShotEventListener();
	OneShotEventListener eventListener3 = new OneShotEventListener(event2);
	ted.subscribeToEvent(partitionId1, eventListener1a, AllWorkersDoneEvent.class);
	ted.subscribeToEvent(partitionId2, eventListener1b, AllWorkersDoneEvent.class);
	ted.subscribeToEvent(partitionId1, eventListener2, TaskEvent.class);
	ted.subscribeToEvent(partitionId1, eventListener3, TerminationEvent.class);

	assertTrue(ted.publish(partitionId1, event1));
	assertTrue("listener should have fired for AllWorkersDoneEvent", eventListener1a.fired);
	assertFalse("listener should not have fired for AllWorkersDoneEvent", eventListener3.fired);

	// publish another event, verify that only the right subscriber is called
	assertTrue(ted.publish(partitionId1, event2));
	assertTrue("listener should have fired for TerminationEvent", eventListener3.fired);
}
 

private SuperstepBarrier initSuperstepBarrier() {
	SuperstepBarrier barrier = new SuperstepBarrier(getUserCodeClassLoader());
	TaskEventDispatcher taskEventDispatcher = getEnvironment().getTaskEventDispatcher();
	ResultPartitionID partitionId = toSyncPartitionId;
	taskEventDispatcher.subscribeToEvent(partitionId, barrier, AllWorkersDoneEvent.class);
	taskEventDispatcher.subscribeToEvent(partitionId, barrier, TerminationEvent.class);
	return barrier;
}
 

@Test
public void clearAll() throws Exception {
	ResultPartitionID partitionId = new ResultPartitionID();
	TaskEventDispatcher ted = new TaskEventDispatcher();
	ted.registerPartition(partitionId);

	//noinspection unchecked
	ZeroShotEventListener eventListener1 = new ZeroShotEventListener();
	ted.subscribeToEvent(partitionId, eventListener1, AllWorkersDoneEvent.class);

	ted.clearAll();

	assertFalse(ted.publish(partitionId, new AllWorkersDoneEvent()));
}
 

@Test
public void unregisterPartition() throws Exception {
	ResultPartitionID partitionId1 = new ResultPartitionID();
	ResultPartitionID partitionId2 = new ResultPartitionID();
	TaskEventDispatcher ted = new TaskEventDispatcher();

	AllWorkersDoneEvent event = new AllWorkersDoneEvent();
	assertFalse(ted.publish(partitionId1, event));

	ted.registerPartition(partitionId1);
	ted.registerPartition(partitionId2);

	OneShotEventListener eventListener1a = new OneShotEventListener(event);
	ZeroShotEventListener eventListener1b = new ZeroShotEventListener();
	OneShotEventListener eventListener2 = new OneShotEventListener(event);
	ted.subscribeToEvent(partitionId1, eventListener1a, AllWorkersDoneEvent.class);
	ted.subscribeToEvent(partitionId2, eventListener1b, AllWorkersDoneEvent.class);
	ted.subscribeToEvent(partitionId1, eventListener2, AllWorkersDoneEvent.class);

	ted.unregisterPartition(partitionId2);

	// publish something for partitionId1 triggering all according listeners
	assertTrue(ted.publish(partitionId1, event));
	assertTrue("listener should have fired for AllWorkersDoneEvent", eventListener1a.fired);
	assertTrue("listener should have fired for AllWorkersDoneEvent", eventListener2.fired);

	// now publish something for partitionId2 which should not trigger any listeners
	assertFalse(ted.publish(partitionId2, event));
}
 

/**
 * Tests {@link TaskEventDispatcher#publish(ResultPartitionID, TaskEvent)} and {@link TaskEventDispatcher#subscribeToEvent(ResultPartitionID, EventListener, Class)} methods.
 */
@Test
public void publishSubscribe() throws Exception {
	ResultPartitionID partitionId1 = new ResultPartitionID();
	ResultPartitionID partitionId2 = new ResultPartitionID();
	TaskEventDispatcher ted = new TaskEventDispatcher();

	AllWorkersDoneEvent event1 = new AllWorkersDoneEvent();
	TerminationEvent event2 = new TerminationEvent();
	assertFalse(ted.publish(partitionId1, event1));

	ted.registerPartition(partitionId1);
	ted.registerPartition(partitionId2);

	// no event listener subscribed yet, but the event is forwarded to a TaskEventHandler
	assertTrue(ted.publish(partitionId1, event1));

	OneShotEventListener eventListener1a = new OneShotEventListener(event1);
	ZeroShotEventListener eventListener1b = new ZeroShotEventListener();
	ZeroShotEventListener eventListener2 = new ZeroShotEventListener();
	OneShotEventListener eventListener3 = new OneShotEventListener(event2);
	ted.subscribeToEvent(partitionId1, eventListener1a, AllWorkersDoneEvent.class);
	ted.subscribeToEvent(partitionId2, eventListener1b, AllWorkersDoneEvent.class);
	ted.subscribeToEvent(partitionId1, eventListener2, TaskEvent.class);
	ted.subscribeToEvent(partitionId1, eventListener3, TerminationEvent.class);

	assertTrue(ted.publish(partitionId1, event1));
	assertTrue("listener should have fired for AllWorkersDoneEvent", eventListener1a.fired);
	assertFalse("listener should not have fired for AllWorkersDoneEvent", eventListener3.fired);

	// publish another event, verify that only the right subscriber is called
	assertTrue(ted.publish(partitionId1, event2));
	assertTrue("listener should have fired for TerminationEvent", eventListener3.fired);
}
 

private SuperstepBarrier initSuperstepBarrier() {
	SuperstepBarrier barrier = new SuperstepBarrier(getUserCodeClassLoader());
	TaskEventDispatcher taskEventDispatcher = getEnvironment().getTaskEventDispatcher();
	ResultPartitionID partitionId = toSyncPartitionId;
	taskEventDispatcher.subscribeToEvent(partitionId, barrier, AllWorkersDoneEvent.class);
	taskEventDispatcher.subscribeToEvent(partitionId, barrier, TerminationEvent.class);
	return barrier;
}
 

@Test
public void clearAll() throws Exception {
	ResultPartitionID partitionId = new ResultPartitionID();
	TaskEventDispatcher ted = new TaskEventDispatcher();
	ted.registerPartition(partitionId);

	//noinspection unchecked
	ZeroShotEventListener eventListener1 = new ZeroShotEventListener();
	ted.subscribeToEvent(partitionId, eventListener1, AllWorkersDoneEvent.class);

	ted.clearAll();

	assertFalse(ted.publish(partitionId, new AllWorkersDoneEvent()));
}
 

@Test
public void unregisterPartition() throws Exception {
	ResultPartitionID partitionId1 = new ResultPartitionID();
	ResultPartitionID partitionId2 = new ResultPartitionID();
	TaskEventDispatcher ted = new TaskEventDispatcher();

	AllWorkersDoneEvent event = new AllWorkersDoneEvent();
	assertFalse(ted.publish(partitionId1, event));

	ted.registerPartition(partitionId1);
	ted.registerPartition(partitionId2);

	OneShotEventListener eventListener1a = new OneShotEventListener(event);
	ZeroShotEventListener eventListener1b = new ZeroShotEventListener();
	OneShotEventListener eventListener2 = new OneShotEventListener(event);
	ted.subscribeToEvent(partitionId1, eventListener1a, AllWorkersDoneEvent.class);
	ted.subscribeToEvent(partitionId2, eventListener1b, AllWorkersDoneEvent.class);
	ted.subscribeToEvent(partitionId1, eventListener2, AllWorkersDoneEvent.class);

	ted.unregisterPartition(partitionId2);

	// publish something for partitionId1 triggering all according listeners
	assertTrue(ted.publish(partitionId1, event));
	assertTrue("listener should have fired for AllWorkersDoneEvent", eventListener1a.fired);
	assertTrue("listener should have fired for AllWorkersDoneEvent", eventListener2.fired);

	// now publish something for partitionId2 which should not trigger any listeners
	assertFalse(ted.publish(partitionId2, event));
}
 

/**
 * Tests {@link TaskEventDispatcher#publish(ResultPartitionID, TaskEvent)} and {@link TaskEventDispatcher#subscribeToEvent(ResultPartitionID, EventListener, Class)} methods.
 */
@Test
public void publishSubscribe() throws Exception {
	ResultPartitionID partitionId1 = new ResultPartitionID();
	ResultPartitionID partitionId2 = new ResultPartitionID();
	TaskEventDispatcher ted = new TaskEventDispatcher();

	AllWorkersDoneEvent event1 = new AllWorkersDoneEvent();
	TerminationEvent event2 = new TerminationEvent();
	assertFalse(ted.publish(partitionId1, event1));

	ted.registerPartition(partitionId1);
	ted.registerPartition(partitionId2);

	// no event listener subscribed yet, but the event is forwarded to a TaskEventHandler
	assertTrue(ted.publish(partitionId1, event1));

	OneShotEventListener eventListener1a = new OneShotEventListener(event1);
	ZeroShotEventListener eventListener1b = new ZeroShotEventListener();
	ZeroShotEventListener eventListener2 = new ZeroShotEventListener();
	OneShotEventListener eventListener3 = new OneShotEventListener(event2);
	ted.subscribeToEvent(partitionId1, eventListener1a, AllWorkersDoneEvent.class);
	ted.subscribeToEvent(partitionId2, eventListener1b, AllWorkersDoneEvent.class);
	ted.subscribeToEvent(partitionId1, eventListener2, TaskEvent.class);
	ted.subscribeToEvent(partitionId1, eventListener3, TerminationEvent.class);

	assertTrue(ted.publish(partitionId1, event1));
	assertTrue("listener should have fired for AllWorkersDoneEvent", eventListener1a.fired);
	assertFalse("listener should not have fired for AllWorkersDoneEvent", eventListener3.fired);

	// publish another event, verify that only the right subscriber is called
	assertTrue(ted.publish(partitionId1, event2));
	assertTrue("listener should have fired for TerminationEvent", eventListener3.fired);
}
 

@Test
public void unregisterPartition() throws Exception {
	ResultPartitionID partitionId1 = new ResultPartitionID();
	ResultPartitionID partitionId2 = new ResultPartitionID();
	TaskEventDispatcher ted = new TaskEventDispatcher();

	AllWorkersDoneEvent event = new AllWorkersDoneEvent();
	assertFalse(ted.publish(partitionId1, event));

	ted.registerPartition(partitionId1);
	ted.registerPartition(partitionId2);

	OneShotEventListener eventListener1a = new OneShotEventListener(event);
	ZeroShotEventListener eventListener1b = new ZeroShotEventListener();
	OneShotEventListener eventListener2 = new OneShotEventListener(event);
	ted.subscribeToEvent(partitionId1, eventListener1a, AllWorkersDoneEvent.class);
	ted.subscribeToEvent(partitionId2, eventListener1b, AllWorkersDoneEvent.class);
	ted.subscribeToEvent(partitionId1, eventListener2, AllWorkersDoneEvent.class);

	ted.unregisterPartition(partitionId2);

	// publish something for partitionId1 triggering all according listeners
	assertTrue(ted.publish(partitionId1, event));
	assertTrue("listener should have fired for AllWorkersDoneEvent", eventListener1a.fired);
	assertTrue("listener should have fired for AllWorkersDoneEvent", eventListener2.fired);

	// now publish something for partitionId2 which should not trigger any listeners
	assertFalse(ted.publish(partitionId2, event));
}
 

private SuperstepBarrier initSuperstepBarrier() {
	SuperstepBarrier barrier = new SuperstepBarrier(getUserCodeClassLoader());
	TaskEventDispatcher taskEventDispatcher = getEnvironment().getTaskEventDispatcher();
	ResultPartitionID partitionId = toSyncPartitionId;
	taskEventDispatcher.subscribeToEvent(partitionId, barrier, AllWorkersDoneEvent.class);
	taskEventDispatcher.subscribeToEvent(partitionId, barrier, TerminationEvent.class);
	return barrier;
}
 

@Test
public void clearAll() throws Exception {
	ResultPartitionID partitionId = new ResultPartitionID();
	TaskEventDispatcher ted = new TaskEventDispatcher();
	ted.registerPartition(partitionId);

	//noinspection unchecked
	ZeroShotEventListener eventListener1 = new ZeroShotEventListener();
	ted.subscribeToEvent(partitionId, eventListener1, AllWorkersDoneEvent.class);

	ted.clearAll();

	assertFalse(ted.publish(partitionId, new AllWorkersDoneEvent()));
}
 

@Test
public void syncAllWorkersDone() throws InterruptedException {
	for (int n = 0; n < 20; n++) {
		sync(new AllWorkersDoneEvent());
	}
}
 

@Test
public void syncAllWorkersDone() throws InterruptedException {
	for (int n = 0; n < 20; n++) {
		sync(new AllWorkersDoneEvent());
	}
}
 

@Override
public void invoke() throws Exception {
	this.headEventReader = new MutableRecordReader<>(
			getEnvironment().getInputGate(0),
			getEnvironment().getTaskManagerInfo().getTmpDirectories());

	TaskConfig taskConfig = new TaskConfig(getTaskConfiguration());

	// store all aggregators
	this.aggregators = new HashMap<>();
	for (AggregatorWithName<?> aggWithName : taskConfig.getIterationAggregators(getUserCodeClassLoader())) {
		aggregators.put(aggWithName.getName(), aggWithName.getAggregator());
	}

	// store the aggregator convergence criterion
	if (taskConfig.usesConvergenceCriterion()) {
		convergenceCriterion = taskConfig.getConvergenceCriterion(getUserCodeClassLoader());
		convergenceAggregatorName = taskConfig.getConvergenceCriterionAggregatorName();
		Preconditions.checkNotNull(convergenceAggregatorName);
	}

	// store the default aggregator convergence criterion
	if (taskConfig.usesImplicitConvergenceCriterion()) {
		implicitConvergenceCriterion = taskConfig.getImplicitConvergenceCriterion(getUserCodeClassLoader());
		implicitConvergenceAggregatorName = taskConfig.getImplicitConvergenceCriterionAggregatorName();
		Preconditions.checkNotNull(implicitConvergenceAggregatorName);
	}

	maxNumberOfIterations = taskConfig.getNumberOfIterations();

	// set up the event handler
	int numEventsTillEndOfSuperstep = taskConfig.getNumberOfEventsUntilInterruptInIterativeGate(0);
	eventHandler = new SyncEventHandler(numEventsTillEndOfSuperstep, aggregators,
			getEnvironment().getUserClassLoader());
	headEventReader.registerTaskEventListener(eventHandler, WorkerDoneEvent.class);

	IntValue dummy = new IntValue();

	while (!terminationRequested()) {

		if (log.isInfoEnabled()) {
			log.info(formatLogString("starting iteration [" + currentIteration + "]"));
		}

		// this call listens for events until the end-of-superstep is reached
		readHeadEventChannel(dummy);

		if (log.isInfoEnabled()) {
			log.info(formatLogString("finishing iteration [" + currentIteration + "]"));
		}

		if (checkForConvergence()) {
			if (log.isInfoEnabled()) {
				log.info(formatLogString("signaling that all workers are to terminate in iteration ["
					+ currentIteration + "]"));
			}

			requestTermination();
			sendToAllWorkers(new TerminationEvent());
		} else {
			if (log.isInfoEnabled()) {
				log.info(formatLogString("signaling that all workers are done in iteration [" + currentIteration
					+ "]"));
			}

			AllWorkersDoneEvent allWorkersDoneEvent = new AllWorkersDoneEvent(aggregators);
			sendToAllWorkers(allWorkersDoneEvent);

			// reset all aggregators
			for (Aggregator<?> agg : aggregators.values()) {
				agg.reset();
			}
			currentIteration++;
		}
	}
}
 

@Override
public void invoke() throws Exception {
	this.headEventReader = new MutableRecordReader<>(
			getEnvironment().getInputGate(0),
			getEnvironment().getTaskManagerInfo().getTmpDirectories());

	TaskConfig taskConfig = new TaskConfig(getTaskConfiguration());

	// store all aggregators
	this.aggregators = new HashMap<>();
	for (AggregatorWithName<?> aggWithName : taskConfig.getIterationAggregators(getUserCodeClassLoader())) {
		aggregators.put(aggWithName.getName(), aggWithName.getAggregator());
	}

	// store the aggregator convergence criterion
	if (taskConfig.usesConvergenceCriterion()) {
		convergenceCriterion = taskConfig.getConvergenceCriterion(getUserCodeClassLoader());
		convergenceAggregatorName = taskConfig.getConvergenceCriterionAggregatorName();
		Preconditions.checkNotNull(convergenceAggregatorName);
	}

	// store the default aggregator convergence criterion
	if (taskConfig.usesImplicitConvergenceCriterion()) {
		implicitConvergenceCriterion = taskConfig.getImplicitConvergenceCriterion(getUserCodeClassLoader());
		implicitConvergenceAggregatorName = taskConfig.getImplicitConvergenceCriterionAggregatorName();
		Preconditions.checkNotNull(implicitConvergenceAggregatorName);
	}

	maxNumberOfIterations = taskConfig.getNumberOfIterations();

	// set up the event handler
	int numEventsTillEndOfSuperstep = taskConfig.getNumberOfEventsUntilInterruptInIterativeGate(0);
	eventHandler = new SyncEventHandler(numEventsTillEndOfSuperstep, aggregators,
			getEnvironment().getUserClassLoader());
	headEventReader.registerTaskEventListener(eventHandler, WorkerDoneEvent.class);

	IntValue dummy = new IntValue();

	while (!terminationRequested()) {

		if (log.isInfoEnabled()) {
			log.info(formatLogString("starting iteration [" + currentIteration + "]"));
		}

		// this call listens for events until the end-of-superstep is reached
		readHeadEventChannel(dummy);

		if (log.isInfoEnabled()) {
			log.info(formatLogString("finishing iteration [" + currentIteration + "]"));
		}

		if (checkForConvergence()) {
			if (log.isInfoEnabled()) {
				log.info(formatLogString("signaling that all workers are to terminate in iteration ["
					+ currentIteration + "]"));
			}

			requestTermination();
			sendToAllWorkers(new TerminationEvent());
		} else {
			if (log.isInfoEnabled()) {
				log.info(formatLogString("signaling that all workers are done in iteration [" + currentIteration
					+ "]"));
			}

			AllWorkersDoneEvent allWorkersDoneEvent = new AllWorkersDoneEvent(aggregators);
			sendToAllWorkers(allWorkersDoneEvent);

			// reset all aggregators
			for (Aggregator<?> agg : aggregators.values()) {
				agg.reset();
			}
			currentIteration++;
		}
	}
}
 

@Test
public void syncAllWorkersDone() throws InterruptedException {
	for (int n = 0; n < 20; n++) {
		sync(new AllWorkersDoneEvent());
	}
}
 

@Override
public void invoke() throws Exception {
	this.headEventReader = new MutableRecordReader<>(
			getEnvironment().getInputGate(0),
			getEnvironment().getTaskManagerInfo().getTmpDirectories());

	TaskConfig taskConfig = new TaskConfig(getTaskConfiguration());

	// store all aggregators
	this.aggregators = new HashMap<>();
	for (AggregatorWithName<?> aggWithName : taskConfig.getIterationAggregators(getUserCodeClassLoader())) {
		aggregators.put(aggWithName.getName(), aggWithName.getAggregator());
	}

	// store the aggregator convergence criterion
	if (taskConfig.usesConvergenceCriterion()) {
		convergenceCriterion = taskConfig.getConvergenceCriterion(getUserCodeClassLoader());
		convergenceAggregatorName = taskConfig.getConvergenceCriterionAggregatorName();
		Preconditions.checkNotNull(convergenceAggregatorName);
	}

	// store the default aggregator convergence criterion
	if (taskConfig.usesImplicitConvergenceCriterion()) {
		implicitConvergenceCriterion = taskConfig.getImplicitConvergenceCriterion(getUserCodeClassLoader());
		implicitConvergenceAggregatorName = taskConfig.getImplicitConvergenceCriterionAggregatorName();
		Preconditions.checkNotNull(implicitConvergenceAggregatorName);
	}

	maxNumberOfIterations = taskConfig.getNumberOfIterations();

	// set up the event handler
	int numEventsTillEndOfSuperstep = taskConfig.getNumberOfEventsUntilInterruptInIterativeGate(0);
	eventHandler = new SyncEventHandler(numEventsTillEndOfSuperstep, aggregators,
			getEnvironment().getUserClassLoader());
	headEventReader.registerTaskEventListener(eventHandler, WorkerDoneEvent.class);

	IntValue dummy = new IntValue();

	while (!terminationRequested()) {

		if (log.isInfoEnabled()) {
			log.info(formatLogString("starting iteration [" + currentIteration + "]"));
		}

		// this call listens for events until the end-of-superstep is reached
		readHeadEventChannel(dummy);

		if (log.isInfoEnabled()) {
			log.info(formatLogString("finishing iteration [" + currentIteration + "]"));
		}

		if (checkForConvergence()) {
			if (log.isInfoEnabled()) {
				log.info(formatLogString("signaling that all workers are to terminate in iteration ["
					+ currentIteration + "]"));
			}

			requestTermination();
			sendToAllWorkers(new TerminationEvent());
		} else {
			if (log.isInfoEnabled()) {
				log.info(formatLogString("signaling that all workers are done in iteration [" + currentIteration
					+ "]"));
			}

			AllWorkersDoneEvent allWorkersDoneEvent = new AllWorkersDoneEvent(aggregators);
			sendToAllWorkers(allWorkersDoneEvent);

			// reset all aggregators
			for (Aggregator<?> agg : aggregators.values()) {
				agg.reset();
			}
			currentIteration++;
		}
	}
}