org.apache.flink.api.common.aggregators.Aggregator Java Examples

private void onWorkerDoneEvent(WorkerDoneEvent workerDoneEvent) {
	if (this.endOfSuperstep) {
		throw new RuntimeException("Encountered WorderDoneEvent when still in End-of-Superstep status.");
	}

	workerDoneEventCounter++;

	String[] aggNames = workerDoneEvent.getAggregatorNames();
	Value[] aggregates = workerDoneEvent.getAggregates(userCodeClassLoader);

	if (aggNames.length != aggregates.length) {
		throw new RuntimeException("Inconsistent WorkerDoneEvent received!");
	}

	for (int i = 0; i < aggNames.length; i++) {
		@SuppressWarnings("unchecked")
		Aggregator<Value> aggregator = (Aggregator<Value>) this.aggregators.get(aggNames[i]);
		aggregator.aggregate(aggregates[i]);
	}

	if (workerDoneEventCounter % numberOfEventsUntilEndOfSuperstep == 0) {
		endOfSuperstep = true;
		Thread.currentThread().interrupt();
	}
}
 

/**
 * Helper method which sets up an iteration with the given vertex value(either simple or with degrees).
 *
 * @param iteration
 */

private void setUpIteration(DeltaIteration<?, ?> iteration) {

	// set up the iteration operator
	if (this.configuration != null) {

		iteration.name(this.configuration.getName("Scatter-gather iteration (" + gatherFunction + " | " + scatterFunction + ")"));
		iteration.parallelism(this.configuration.getParallelism());
		iteration.setSolutionSetUnManaged(this.configuration.isSolutionSetUnmanagedMemory());

		// register all aggregators
		for (Map.Entry<String, Aggregator<?>> entry : this.configuration.getAggregators().entrySet()) {
			iteration.registerAggregator(entry.getKey(), entry.getValue());
		}
	}
	else {
		// no configuration provided; set default name
		iteration.name("Scatter-gather iteration (" + gatherFunction + " | " + scatterFunction + ")");
	}
}
 

/**
 * Helper method which sets up an iteration with the given vertex value.
 *
 * @param iteration
 */

private void setUpIteration(DeltaIteration<?, ?> iteration) {

	// set up the iteration operator
	if (this.configuration != null) {

		iteration.name(this.configuration.getName("Vertex-centric iteration (" + computeFunction + ")"));
		iteration.parallelism(this.configuration.getParallelism());
		iteration.setSolutionSetUnManaged(this.configuration.isSolutionSetUnmanagedMemory());

		// register all aggregators
		for (Map.Entry<String, Aggregator<?>> entry : this.configuration.getAggregators().entrySet()) {
			iteration.registerAggregator(entry.getKey(), entry.getValue());
		}
	}
	else {
		// no configuration provided; set default name
		iteration.name("Vertex-centric iteration (" + computeFunction + ")");
	}
}
 

protected IterationEventWithAggregators(Map<String, Aggregator<?>> aggregators) {
	int num = aggregators.size();
	if (num == 0) {
		this.aggNames = NO_STRINGS;
		this.aggregates = NO_VALUES;
	} else {
		this.aggNames = new String[num];
		this.aggregates = new Value[num];

		int i = 0;
		for (Map.Entry<String, Aggregator<?>> entry : aggregators.entrySet()) {
			this.aggNames[i] = entry.getKey();
			this.aggregates[i] = entry.getValue().getAggregate();
			i++;
		}
	}
}
 

protected IterationEventWithAggregators(Map<String, Aggregator<?>> aggregators) {
	int num = aggregators.size();
	if (num == 0) {
		this.aggNames = NO_STRINGS;
		this.aggregates = NO_VALUES;
	} else {
		this.aggNames = new String[num];
		this.aggregates = new Value[num];

		int i = 0;
		for (Map.Entry<String, Aggregator<?>> entry : aggregators.entrySet()) {
			this.aggNames[i] = entry.getKey();
			this.aggregates[i] = entry.getValue().getAggregate();
			i++;
		}
	}
}
 

private void onWorkerDoneEvent(WorkerDoneEvent workerDoneEvent) {
	if (this.endOfSuperstep) {
		throw new RuntimeException("Encountered WorderDoneEvent when still in End-of-Superstep status.");
	}

	workerDoneEventCounter++;

	String[] aggNames = workerDoneEvent.getAggregatorNames();
	Value[] aggregates = workerDoneEvent.getAggregates(userCodeClassLoader);

	if (aggNames.length != aggregates.length) {
		throw new RuntimeException("Inconsistent WorkerDoneEvent received!");
	}

	for (int i = 0; i < aggNames.length; i++) {
		@SuppressWarnings("unchecked")
		Aggregator<Value> aggregator = (Aggregator<Value>) this.aggregators.get(aggNames[i]);
		aggregator.aggregate(aggregates[i]);
	}

	if (workerDoneEventCounter % numberOfEventsUntilEndOfSuperstep == 0) {
		endOfSuperstep = true;
		Thread.currentThread().interrupt();
	}
}
 

private void onWorkerDoneEvent(WorkerDoneEvent workerDoneEvent) {
	if (this.endOfSuperstep) {
		throw new RuntimeException("Encountered WorderDoneEvent when still in End-of-Superstep status.");
	}

	workerDoneEventCounter++;

	String[] aggNames = workerDoneEvent.getAggregatorNames();
	Value[] aggregates = workerDoneEvent.getAggregates(userCodeClassLoader);

	if (aggNames.length != aggregates.length) {
		throw new RuntimeException("Inconsistent WorkerDoneEvent received!");
	}

	for (int i = 0; i < aggNames.length; i++) {
		@SuppressWarnings("unchecked")
		Aggregator<Value> aggregator = (Aggregator<Value>) this.aggregators.get(aggNames[i]);
		aggregator.aggregate(aggregates[i]);
	}

	if (workerDoneEventCounter % numberOfEventsUntilEndOfSuperstep == 0) {
		endOfSuperstep = true;
		Thread.currentThread().interrupt();
	}
}
 

/**
 * Helper method which sets up an iteration with the given vertex value.
 *
 * @param iteration
 */

private void setUpIteration(DeltaIteration<?, ?> iteration) {

	// set up the iteration operator
	if (this.configuration != null) {

		iteration.name(this.configuration.getName("Vertex-centric iteration (" + computeFunction + ")"));
		iteration.parallelism(this.configuration.getParallelism());
		iteration.setSolutionSetUnManaged(this.configuration.isSolutionSetUnmanagedMemory());

		// register all aggregators
		for (Map.Entry<String, Aggregator<?>> entry : this.configuration.getAggregators().entrySet()) {
			iteration.registerAggregator(entry.getKey(), entry.getValue());
		}
	}
	else {
		// no configuration provided; set default name
		iteration.name("Vertex-centric iteration (" + computeFunction + ")");
	}
}
 

/**
 * Helper method which sets up an iteration with the given vertex value(either simple or with degrees).
 *
 * @param iteration
 */

private void setUpIteration(DeltaIteration<?, ?> iteration) {

	// set up the iteration operator
	if (this.configuration != null) {

		iteration.name(this.configuration.getName("Scatter-gather iteration (" + gatherFunction + " | " + scatterFunction + ")"));
		iteration.parallelism(this.configuration.getParallelism());
		iteration.setSolutionSetUnManaged(this.configuration.isSolutionSetUnmanagedMemory());

		// register all aggregators
		for (Map.Entry<String, Aggregator<?>> entry : this.configuration.getAggregators().entrySet()) {
			iteration.registerAggregator(entry.getKey(), entry.getValue());
		}
	}
	else {
		// no configuration provided; set default name
		iteration.name("Scatter-gather iteration (" + gatherFunction + " | " + scatterFunction + ")");
	}
}
 

/**
 * Helper method which sets up an iteration with the given vertex value(either simple or with degrees).
 *
 * @param iteration
 */

private void setUpIteration(DeltaIteration<?, ?> iteration) {

	// set up the iteration operator
	if (this.configuration != null) {

		iteration.name(this.configuration.getName("Scatter-gather iteration (" + gatherFunction + " | " + scatterFunction + ")"));
		iteration.parallelism(this.configuration.getParallelism());
		iteration.setSolutionSetUnManaged(this.configuration.isSolutionSetUnmanagedMemory());

		// register all aggregators
		for (Map.Entry<String, Aggregator<?>> entry : this.configuration.getAggregators().entrySet()) {
			iteration.registerAggregator(entry.getKey(), entry.getValue());
		}
	}
	else {
		// no configuration provided; set default name
		iteration.name("Scatter-gather iteration (" + gatherFunction + " | " + scatterFunction + ")");
	}
}
 

public void updateGlobalAggregatesAndReset(String[] names, Value[] aggregates) {
	if (names == null || aggregates == null || names.length != aggregates.length) {
		throw new IllegalArgumentException();
	}

	// add global aggregates
	for (int i = 0; i < names.length; i++) {
		this.previousGlobalAggregate.put(names[i], aggregates[i]);
	}

	// reset all aggregators
	for (Aggregator<?> agg : this.aggregators.values()) {
		agg.reset();
	}
}
 

public RuntimeAggregatorRegistry(Collection<AggregatorWithName<?>> aggs) {
	this.aggregators = new HashMap<String, Aggregator<?>>();
	this.previousGlobalAggregate = new HashMap<String, Value>();

	for (AggregatorWithName<?> agg : aggs) {
		this.aggregators.put(agg.getName(), agg.getAggregator());
	}
}
 

public CollectionExecutor(ExecutionConfig executionConfig) {
	this.executionConfig = executionConfig;
	
	this.intermediateResults = new HashMap<Operator<?>, List<?>>();
	this.accumulators = new HashMap<String, Accumulator<?,?>>();
	this.previousAggregates = new HashMap<String, Value>();
	this.aggregators = new HashMap<String, Aggregator<?>>();
	this.cachedFiles = new HashMap<String, Future<Path>>();
	this.userCodeClassLoader = Thread.currentThread().getContextClassLoader();
}
 

public void addIterationAggregator(String name, Aggregator<?> aggregator) {
	int num = this.config.getInteger(ITERATION_NUM_AGGREGATORS, 0);
	this.config.setString(ITERATION_AGGREGATOR_NAME_PREFIX + num, name);
	try {
			InstantiationUtil.writeObjectToConfig(aggregator, this.config, ITERATION_AGGREGATOR_PREFIX + num);
	} catch (IOException e) {
			throw new RuntimeException("Error while writing the aggregator object to the task configuration.");
	}
	this.config.setInteger(ITERATION_NUM_AGGREGATORS, num + 1);
}
 

public CollectionExecutor(ExecutionConfig executionConfig) {
	this.executionConfig = executionConfig;
	
	this.intermediateResults = new HashMap<Operator<?>, List<?>>();
	this.accumulators = new HashMap<String, Accumulator<?,?>>();
	this.previousAggregates = new HashMap<String, Value>();
	this.aggregators = new HashMap<String, Aggregator<?>>();
	this.cachedFiles = new HashMap<String, Future<Path>>();
	this.userCodeClassLoader = Thread.currentThread().getContextClassLoader();
}
 

public RuntimeAggregatorRegistry(Collection<AggregatorWithName<?>> aggs) {
	this.aggregators = new HashMap<String, Aggregator<?>>();
	this.previousGlobalAggregate = new HashMap<String, Value>();

	for (AggregatorWithName<?> agg : aggs) {
		this.aggregators.put(agg.getName(), agg.getAggregator());
	}
}
 

public RuntimeAggregatorRegistry(Collection<AggregatorWithName<?>> aggs) {
	this.aggregators = new HashMap<String, Aggregator<?>>();
	this.previousGlobalAggregate = new HashMap<String, Value>();

	for (AggregatorWithName<?> agg : aggs) {
		this.aggregators.put(agg.getName(), agg.getAggregator());
	}
}
 

public void updateGlobalAggregatesAndReset(String[] names, Value[] aggregates) {
	if (names == null || aggregates == null || names.length != aggregates.length) {
		throw new IllegalArgumentException();
	}

	// add global aggregates
	for (int i = 0; i < names.length; i++) {
		this.previousGlobalAggregate.put(names[i], aggregates[i]);
	}

	// reset all aggregators
	for (Aggregator<?> agg : this.aggregators.values()) {
		agg.reset();
	}
}
 

public CollectionExecutor(ExecutionConfig executionConfig) {
	this.executionConfig = executionConfig;
	
	this.intermediateResults = new HashMap<Operator<?>, List<?>>();
	this.accumulators = new HashMap<String, Accumulator<?,?>>();
	this.previousAggregates = new HashMap<String, Value>();
	this.aggregators = new HashMap<String, Aggregator<?>>();
	this.cachedFiles = new HashMap<String, Future<Path>>();
	this.userCodeClassLoader = Thread.currentThread().getContextClassLoader();
}
 

public void addIterationAggregator(String name, Aggregator<?> aggregator) {
	int num = this.config.getInteger(ITERATION_NUM_AGGREGATORS, 0);
	this.config.setString(ITERATION_AGGREGATOR_NAME_PREFIX + num, name);
	try {
			InstantiationUtil.writeObjectToConfig(aggregator, this.config, ITERATION_AGGREGATOR_PREFIX + num);
	} catch (IOException e) {
			throw new RuntimeException("Error while writing the aggregator object to the task configuration.");
	}
	this.config.setInteger(ITERATION_NUM_AGGREGATORS, num + 1);
}
 

@SuppressWarnings("unchecked")
public <T extends Aggregator<?>> T getAggregator(String name) {
	return (T) this.aggregators.get(name);
}
 

@Override
public <T extends Aggregator<?>> T getIterationAggregator(String name) {
	return getIterationAggregators().<T>getAggregator(name);
}
 

@Override
public void invoke() throws Exception {
	this.headEventReader = new MutableRecordReader<>(
			getEnvironment().getInputGate(0),
			getEnvironment().getTaskManagerInfo().getTmpDirectories());

	TaskConfig taskConfig = new TaskConfig(getTaskConfiguration());

	// store all aggregators
	this.aggregators = new HashMap<>();
	for (AggregatorWithName<?> aggWithName : taskConfig.getIterationAggregators(getUserCodeClassLoader())) {
		aggregators.put(aggWithName.getName(), aggWithName.getAggregator());
	}

	// store the aggregator convergence criterion
	if (taskConfig.usesConvergenceCriterion()) {
		convergenceCriterion = taskConfig.getConvergenceCriterion(getUserCodeClassLoader());
		convergenceAggregatorName = taskConfig.getConvergenceCriterionAggregatorName();
		Preconditions.checkNotNull(convergenceAggregatorName);
	}

	// store the default aggregator convergence criterion
	if (taskConfig.usesImplicitConvergenceCriterion()) {
		implicitConvergenceCriterion = taskConfig.getImplicitConvergenceCriterion(getUserCodeClassLoader());
		implicitConvergenceAggregatorName = taskConfig.getImplicitConvergenceCriterionAggregatorName();
		Preconditions.checkNotNull(implicitConvergenceAggregatorName);
	}

	maxNumberOfIterations = taskConfig.getNumberOfIterations();

	// set up the event handler
	int numEventsTillEndOfSuperstep = taskConfig.getNumberOfEventsUntilInterruptInIterativeGate(0);
	eventHandler = new SyncEventHandler(numEventsTillEndOfSuperstep, aggregators,
			getEnvironment().getUserClassLoader());
	headEventReader.registerTaskEventListener(eventHandler, WorkerDoneEvent.class);

	IntValue dummy = new IntValue();

	while (!terminationRequested()) {

		if (log.isInfoEnabled()) {
			log.info(formatLogString("starting iteration [" + currentIteration + "]"));
		}

		// this call listens for events until the end-of-superstep is reached
		readHeadEventChannel(dummy);

		if (log.isInfoEnabled()) {
			log.info(formatLogString("finishing iteration [" + currentIteration + "]"));
		}

		if (checkForConvergence()) {
			if (log.isInfoEnabled()) {
				log.info(formatLogString("signaling that all workers are to terminate in iteration ["
					+ currentIteration + "]"));
			}

			requestTermination();
			sendToAllWorkers(new TerminationEvent());
		} else {
			if (log.isInfoEnabled()) {
				log.info(formatLogString("signaling that all workers are done in iteration [" + currentIteration
					+ "]"));
			}

			AllWorkersDoneEvent allWorkersDoneEvent = new AllWorkersDoneEvent(aggregators);
			sendToAllWorkers(allWorkersDoneEvent);

			// reset all aggregators
			for (Aggregator<?> agg : aggregators.values()) {
				agg.reset();
			}
			currentIteration++;
		}
	}
}
 

@Test
public void testSerializationOfEventWithAggregateValues() {
	StringValue stringValue = new StringValue("test string");
	LongValue longValue = new LongValue(68743254);

	String stringValueName = "stringValue";
	String longValueName = "longValue";

	Aggregator<StringValue> stringAgg = new TestAggregator<StringValue>(stringValue);
	Aggregator<LongValue> longAgg = new TestAggregator<LongValue>(longValue);

	Map<String, Aggregator<?>> aggMap = new HashMap<String,  Aggregator<?>>();
	aggMap.put(stringValueName, stringAgg);
	aggMap.put(longValueName, longAgg);

	Set<String> allNames = new HashSet<String>();
	allNames.add(stringValueName);
	allNames.add(longValueName);

	Set<Value> allVals = new HashSet<Value>();
	allVals.add(stringValue);
	allVals.add(longValue);

	// run the serialization
	AllWorkersDoneEvent e = new AllWorkersDoneEvent(aggMap);
	IterationEventWithAggregators deserialized = pipeThroughSerialization(e);

	// verify the result
	String[] names = deserialized.getAggregatorNames();
	Value[] aggregates = deserialized.getAggregates(cl);

	Assert.assertEquals(allNames.size(), names.length);
	Assert.assertEquals(allVals.size(), aggregates.length);

	// check that all the correct names and values are returned
	for (String s : names) {
		allNames.remove(s);
	}
	for (Value v : aggregates) {
		allVals.remove(v);
	}

	Assert.assertTrue(allNames.isEmpty());
	Assert.assertTrue(allVals.isEmpty());
}
 

public SyncEventHandler(int numberOfEventsUntilEndOfSuperstep, Map<String, Aggregator<?>> aggregators, ClassLoader userCodeClassLoader) {
	Preconditions.checkArgument(numberOfEventsUntilEndOfSuperstep > 0);
	this.userCodeClassLoader = userCodeClassLoader;
	this.numberOfEventsUntilEndOfSuperstep = numberOfEventsUntilEndOfSuperstep;
	this.aggregators = aggregators;
}
 

public Map<String, Aggregator<?>> getAllAggregators() {
	return this.aggregators;
}
 

@PublicEvolving
<T extends Aggregator<?>> T getIterationAggregator(String name);
 

@SuppressWarnings("unchecked")
@Override
public <T extends Aggregator<?>> T getIterationAggregator(String name) {
	return (T) aggregators.get(name);
}
 

@SuppressWarnings("unchecked")
private <T> List<T> executeBulkIteration(BulkIterationBase<?> iteration) throws Exception {
	Operator<?> inputOp = iteration.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The iteration " + iteration.getName() + " has no input (initial partial solution).");
	}
	if (iteration.getNextPartialSolution() == null) {
		throw new InvalidProgramException("The iteration " + iteration.getName() + " has no next partial solution defined (is not closed).");
	}
	
	List<T> inputData = (List<T>) execute(inputOp);
	
	// get the operators that are iterative
	Set<Operator<?>> dynamics = new LinkedHashSet<Operator<?>>();
	DynamicPathCollector dynCollector = new DynamicPathCollector(dynamics);
	iteration.getNextPartialSolution().accept(dynCollector);
	if (iteration.getTerminationCriterion() != null) {
		iteration.getTerminationCriterion().accept(dynCollector);
	}
	
	// register the aggregators
	for (AggregatorWithName<?> a : iteration.getAggregators().getAllRegisteredAggregators()) {
		aggregators.put(a.getName(), a.getAggregator());
	}
	
	String convCriterionAggName = iteration.getAggregators().getConvergenceCriterionAggregatorName();
	ConvergenceCriterion<Value> convCriterion = (ConvergenceCriterion<Value>) iteration.getAggregators().getConvergenceCriterion();
	
	List<T> currentResult = inputData;
	
	final int maxIterations = iteration.getMaximumNumberOfIterations();
	
	for (int superstep = 1; superstep <= maxIterations; superstep++) {
		
		// set the input to the current partial solution
		this.intermediateResults.put(iteration.getPartialSolution(), currentResult);

		// set the superstep number
		iterationSuperstep = superstep;

		// grab the current iteration result
		currentResult = (List<T>) execute(iteration.getNextPartialSolution(), superstep);

		// evaluate the termination criterion
		if (iteration.getTerminationCriterion() != null) {
			execute(iteration.getTerminationCriterion(), superstep);
		}
		
		// evaluate the aggregator convergence criterion
		if (convCriterion != null && convCriterionAggName != null) {
			Value v = aggregators.get(convCriterionAggName).getAggregate();
			if (convCriterion.isConverged(superstep, v)) {
				break;
			}
		}
		
		// clear the dynamic results
		for (Operator<?> o : dynamics) {
			intermediateResults.remove(o);
		}
		
		// set the previous iteration's aggregates and reset the aggregators
		for (Map.Entry<String, Aggregator<?>> e : aggregators.entrySet()) {
			previousAggregates.put(e.getKey(), e.getValue().getAggregate());
			e.getValue().reset();
		}
	}
	
	previousAggregates.clear();
	aggregators.clear();
	
	return currentResult;
}
 

@Test
public void testSerializationOfEventWithAggregateValues() {
	StringValue stringValue = new StringValue("test string");
	LongValue longValue = new LongValue(68743254);

	String stringValueName = "stringValue";
	String longValueName = "longValue";

	Aggregator<StringValue> stringAgg = new TestAggregator<StringValue>(stringValue);
	Aggregator<LongValue> longAgg = new TestAggregator<LongValue>(longValue);

	Map<String, Aggregator<?>> aggMap = new HashMap<String,  Aggregator<?>>();
	aggMap.put(stringValueName, stringAgg);
	aggMap.put(longValueName, longAgg);

	Set<String> allNames = new HashSet<String>();
	allNames.add(stringValueName);
	allNames.add(longValueName);

	Set<Value> allVals = new HashSet<Value>();
	allVals.add(stringValue);
	allVals.add(longValue);

	// run the serialization
	AllWorkersDoneEvent e = new AllWorkersDoneEvent(aggMap);
	IterationEventWithAggregators deserialized = pipeThroughSerialization(e);

	// verify the result
	String[] names = deserialized.getAggregatorNames();
	Value[] aggregates = deserialized.getAggregates(cl);

	Assert.assertEquals(allNames.size(), names.length);
	Assert.assertEquals(allVals.size(), aggregates.length);

	// check that all the correct names and values are returned
	for (String s : names) {
		allNames.remove(s);
	}
	for (Value v : aggregates) {
		allVals.remove(v);
	}

	Assert.assertTrue(allNames.isEmpty());
	Assert.assertTrue(allVals.isEmpty());
}