org.apache.flink.api.common.aggregators.AggregatorWithName Java Examples

public RuntimeAggregatorRegistry(Collection<AggregatorWithName<?>> aggs) {
	this.aggregators = new HashMap<String, Aggregator<?>>();
	this.previousGlobalAggregate = new HashMap<String, Value>();

	for (AggregatorWithName<?> agg : aggs) {
		this.aggregators.put(agg.getName(), agg.getAggregator());
	}
}
 

public void addIterationAggregators(Collection<AggregatorWithName<?>> aggregators) {
	int num = this.config.getInteger(ITERATION_NUM_AGGREGATORS, 0);
	for (AggregatorWithName<?> awn : aggregators) {
		this.config.setString(ITERATION_AGGREGATOR_NAME_PREFIX + num, awn.getName());
		try {
			InstantiationUtil.writeObjectToConfig(awn.getAggregator(), this.config, ITERATION_AGGREGATOR_PREFIX + num);
		} catch (IOException e) {
			throw new RuntimeException("Error while writing the aggregator object to the task configuration.");
		}
		num++;
	}
	this.config.setInteger(ITERATION_NUM_AGGREGATORS, num);
}
 

public RuntimeAggregatorRegistry(Collection<AggregatorWithName<?>> aggs) {
	this.aggregators = new HashMap<String, Aggregator<?>>();
	this.previousGlobalAggregate = new HashMap<String, Value>();

	for (AggregatorWithName<?> agg : aggs) {
		this.aggregators.put(agg.getName(), agg.getAggregator());
	}
}
 

public void addIterationAggregators(Collection<AggregatorWithName<?>> aggregators) {
	int num = this.config.getInteger(ITERATION_NUM_AGGREGATORS, 0);
	for (AggregatorWithName<?> awn : aggregators) {
		this.config.setString(ITERATION_AGGREGATOR_NAME_PREFIX + num, awn.getName());
		try {
			InstantiationUtil.writeObjectToConfig(awn.getAggregator(), this.config, ITERATION_AGGREGATOR_PREFIX + num);
		} catch (IOException e) {
			throw new RuntimeException("Error while writing the aggregator object to the task configuration.");
		}
		num++;
	}
	this.config.setInteger(ITERATION_NUM_AGGREGATORS, num);
}
 

public RuntimeAggregatorRegistry(Collection<AggregatorWithName<?>> aggs) {
	this.aggregators = new HashMap<String, Aggregator<?>>();
	this.previousGlobalAggregate = new HashMap<String, Value>();

	for (AggregatorWithName<?> agg : aggs) {
		this.aggregators.put(agg.getName(), agg.getAggregator());
	}
}
 

public void addIterationAggregators(Collection<AggregatorWithName<?>> aggregators) {
	int num = this.config.getInteger(ITERATION_NUM_AGGREGATORS, 0);
	for (AggregatorWithName<?> awn : aggregators) {
		this.config.setString(ITERATION_AGGREGATOR_NAME_PREFIX + num, awn.getName());
		try {
			InstantiationUtil.writeObjectToConfig(awn.getAggregator(), this.config, ITERATION_AGGREGATOR_PREFIX + num);
		} catch (IOException e) {
			throw new RuntimeException("Error while writing the aggregator object to the task configuration.");
		}
		num++;
	}
	this.config.setInteger(ITERATION_NUM_AGGREGATORS, num);
}
 

@SuppressWarnings("unchecked")
private <T> List<T> executeBulkIteration(BulkIterationBase<?> iteration) throws Exception {
	Operator<?> inputOp = iteration.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The iteration " + iteration.getName() + " has no input (initial partial solution).");
	}
	if (iteration.getNextPartialSolution() == null) {
		throw new InvalidProgramException("The iteration " + iteration.getName() + " has no next partial solution defined (is not closed).");
	}
	
	List<T> inputData = (List<T>) execute(inputOp);
	
	// get the operators that are iterative
	Set<Operator<?>> dynamics = new LinkedHashSet<Operator<?>>();
	DynamicPathCollector dynCollector = new DynamicPathCollector(dynamics);
	iteration.getNextPartialSolution().accept(dynCollector);
	if (iteration.getTerminationCriterion() != null) {
		iteration.getTerminationCriterion().accept(dynCollector);
	}
	
	// register the aggregators
	for (AggregatorWithName<?> a : iteration.getAggregators().getAllRegisteredAggregators()) {
		aggregators.put(a.getName(), a.getAggregator());
	}
	
	String convCriterionAggName = iteration.getAggregators().getConvergenceCriterionAggregatorName();
	ConvergenceCriterion<Value> convCriterion = (ConvergenceCriterion<Value>) iteration.getAggregators().getConvergenceCriterion();
	
	List<T> currentResult = inputData;
	
	final int maxIterations = iteration.getMaximumNumberOfIterations();
	
	for (int superstep = 1; superstep <= maxIterations; superstep++) {
		
		// set the input to the current partial solution
		this.intermediateResults.put(iteration.getPartialSolution(), currentResult);

		// set the superstep number
		iterationSuperstep = superstep;

		// grab the current iteration result
		currentResult = (List<T>) execute(iteration.getNextPartialSolution(), superstep);

		// evaluate the termination criterion
		if (iteration.getTerminationCriterion() != null) {
			execute(iteration.getTerminationCriterion(), superstep);
		}
		
		// evaluate the aggregator convergence criterion
		if (convCriterion != null && convCriterionAggName != null) {
			Value v = aggregators.get(convCriterionAggName).getAggregate();
			if (convCriterion.isConverged(superstep, v)) {
				break;
			}
		}
		
		// clear the dynamic results
		for (Operator<?> o : dynamics) {
			intermediateResults.remove(o);
		}
		
		// set the previous iteration's aggregates and reset the aggregators
		for (Map.Entry<String, Aggregator<?>> e : aggregators.entrySet()) {
			previousAggregates.put(e.getKey(), e.getValue().getAggregate());
			e.getValue().reset();
		}
	}
	
	previousAggregates.clear();
	aggregators.clear();
	
	return currentResult;
}
 

@Override
public void invoke() throws Exception {
	this.headEventReader = new MutableRecordReader<>(
			getEnvironment().getInputGate(0),
			getEnvironment().getTaskManagerInfo().getTmpDirectories());

	TaskConfig taskConfig = new TaskConfig(getTaskConfiguration());

	// store all aggregators
	this.aggregators = new HashMap<>();
	for (AggregatorWithName<?> aggWithName : taskConfig.getIterationAggregators(getUserCodeClassLoader())) {
		aggregators.put(aggWithName.getName(), aggWithName.getAggregator());
	}

	// store the aggregator convergence criterion
	if (taskConfig.usesConvergenceCriterion()) {
		convergenceCriterion = taskConfig.getConvergenceCriterion(getUserCodeClassLoader());
		convergenceAggregatorName = taskConfig.getConvergenceCriterionAggregatorName();
		Preconditions.checkNotNull(convergenceAggregatorName);
	}

	// store the default aggregator convergence criterion
	if (taskConfig.usesImplicitConvergenceCriterion()) {
		implicitConvergenceCriterion = taskConfig.getImplicitConvergenceCriterion(getUserCodeClassLoader());
		implicitConvergenceAggregatorName = taskConfig.getImplicitConvergenceCriterionAggregatorName();
		Preconditions.checkNotNull(implicitConvergenceAggregatorName);
	}

	maxNumberOfIterations = taskConfig.getNumberOfIterations();

	// set up the event handler
	int numEventsTillEndOfSuperstep = taskConfig.getNumberOfEventsUntilInterruptInIterativeGate(0);
	eventHandler = new SyncEventHandler(numEventsTillEndOfSuperstep, aggregators,
			getEnvironment().getUserClassLoader());
	headEventReader.registerTaskEventListener(eventHandler, WorkerDoneEvent.class);

	IntValue dummy = new IntValue();

	while (!terminationRequested()) {

		if (log.isInfoEnabled()) {
			log.info(formatLogString("starting iteration [" + currentIteration + "]"));
		}

		// this call listens for events until the end-of-superstep is reached
		readHeadEventChannel(dummy);

		if (log.isInfoEnabled()) {
			log.info(formatLogString("finishing iteration [" + currentIteration + "]"));
		}

		if (checkForConvergence()) {
			if (log.isInfoEnabled()) {
				log.info(formatLogString("signaling that all workers are to terminate in iteration ["
					+ currentIteration + "]"));
			}

			requestTermination();
			sendToAllWorkers(new TerminationEvent());
		} else {
			if (log.isInfoEnabled()) {
				log.info(formatLogString("signaling that all workers are done in iteration [" + currentIteration
					+ "]"));
			}

			AllWorkersDoneEvent allWorkersDoneEvent = new AllWorkersDoneEvent(aggregators);
			sendToAllWorkers(allWorkersDoneEvent);

			// reset all aggregators
			for (Aggregator<?> agg : aggregators.values()) {
				agg.reset();
			}
			currentIteration++;
		}
	}
}
 

@SuppressWarnings("unchecked")
private <T> List<T> executeBulkIteration(BulkIterationBase<?> iteration) throws Exception {
	Operator<?> inputOp = iteration.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The iteration " + iteration.getName() + " has no input (initial partial solution).");
	}
	if (iteration.getNextPartialSolution() == null) {
		throw new InvalidProgramException("The iteration " + iteration.getName() + " has no next partial solution defined (is not closed).");
	}
	
	List<T> inputData = (List<T>) execute(inputOp);
	
	// get the operators that are iterative
	Set<Operator<?>> dynamics = new LinkedHashSet<Operator<?>>();
	DynamicPathCollector dynCollector = new DynamicPathCollector(dynamics);
	iteration.getNextPartialSolution().accept(dynCollector);
	if (iteration.getTerminationCriterion() != null) {
		iteration.getTerminationCriterion().accept(dynCollector);
	}
	
	// register the aggregators
	for (AggregatorWithName<?> a : iteration.getAggregators().getAllRegisteredAggregators()) {
		aggregators.put(a.getName(), a.getAggregator());
	}
	
	String convCriterionAggName = iteration.getAggregators().getConvergenceCriterionAggregatorName();
	ConvergenceCriterion<Value> convCriterion = (ConvergenceCriterion<Value>) iteration.getAggregators().getConvergenceCriterion();
	
	List<T> currentResult = inputData;
	
	final int maxIterations = iteration.getMaximumNumberOfIterations();
	
	for (int superstep = 1; superstep <= maxIterations; superstep++) {
		
		// set the input to the current partial solution
		this.intermediateResults.put(iteration.getPartialSolution(), currentResult);

		// set the superstep number
		iterationSuperstep = superstep;

		// grab the current iteration result
		currentResult = (List<T>) execute(iteration.getNextPartialSolution(), superstep);

		// evaluate the termination criterion
		if (iteration.getTerminationCriterion() != null) {
			execute(iteration.getTerminationCriterion(), superstep);
		}
		
		// evaluate the aggregator convergence criterion
		if (convCriterion != null && convCriterionAggName != null) {
			Value v = aggregators.get(convCriterionAggName).getAggregate();
			if (convCriterion.isConverged(superstep, v)) {
				break;
			}
		}
		
		// clear the dynamic results
		for (Operator<?> o : dynamics) {
			intermediateResults.remove(o);
		}
		
		// set the previous iteration's aggregates and reset the aggregators
		for (Map.Entry<String, Aggregator<?>> e : aggregators.entrySet()) {
			previousAggregates.put(e.getKey(), e.getValue().getAggregate());
			e.getValue().reset();
		}
	}
	
	previousAggregates.clear();
	aggregators.clear();
	
	return currentResult;
}
 

@Override
public void invoke() throws Exception {
	this.headEventReader = new MutableRecordReader<>(
			getEnvironment().getInputGate(0),
			getEnvironment().getTaskManagerInfo().getTmpDirectories());

	TaskConfig taskConfig = new TaskConfig(getTaskConfiguration());

	// store all aggregators
	this.aggregators = new HashMap<>();
	for (AggregatorWithName<?> aggWithName : taskConfig.getIterationAggregators(getUserCodeClassLoader())) {
		aggregators.put(aggWithName.getName(), aggWithName.getAggregator());
	}

	// store the aggregator convergence criterion
	if (taskConfig.usesConvergenceCriterion()) {
		convergenceCriterion = taskConfig.getConvergenceCriterion(getUserCodeClassLoader());
		convergenceAggregatorName = taskConfig.getConvergenceCriterionAggregatorName();
		Preconditions.checkNotNull(convergenceAggregatorName);
	}

	// store the default aggregator convergence criterion
	if (taskConfig.usesImplicitConvergenceCriterion()) {
		implicitConvergenceCriterion = taskConfig.getImplicitConvergenceCriterion(getUserCodeClassLoader());
		implicitConvergenceAggregatorName = taskConfig.getImplicitConvergenceCriterionAggregatorName();
		Preconditions.checkNotNull(implicitConvergenceAggregatorName);
	}

	maxNumberOfIterations = taskConfig.getNumberOfIterations();

	// set up the event handler
	int numEventsTillEndOfSuperstep = taskConfig.getNumberOfEventsUntilInterruptInIterativeGate(0);
	eventHandler = new SyncEventHandler(numEventsTillEndOfSuperstep, aggregators,
			getEnvironment().getUserClassLoader());
	headEventReader.registerTaskEventListener(eventHandler, WorkerDoneEvent.class);

	IntValue dummy = new IntValue();

	while (!terminationRequested()) {

		if (log.isInfoEnabled()) {
			log.info(formatLogString("starting iteration [" + currentIteration + "]"));
		}

		// this call listens for events until the end-of-superstep is reached
		readHeadEventChannel(dummy);

		if (log.isInfoEnabled()) {
			log.info(formatLogString("finishing iteration [" + currentIteration + "]"));
		}

		if (checkForConvergence()) {
			if (log.isInfoEnabled()) {
				log.info(formatLogString("signaling that all workers are to terminate in iteration ["
					+ currentIteration + "]"));
			}

			requestTermination();
			sendToAllWorkers(new TerminationEvent());
		} else {
			if (log.isInfoEnabled()) {
				log.info(formatLogString("signaling that all workers are done in iteration [" + currentIteration
					+ "]"));
			}

			AllWorkersDoneEvent allWorkersDoneEvent = new AllWorkersDoneEvent(aggregators);
			sendToAllWorkers(allWorkersDoneEvent);

			// reset all aggregators
			for (Aggregator<?> agg : aggregators.values()) {
				agg.reset();
			}
			currentIteration++;
		}
	}
}
 

@SuppressWarnings("unchecked")
private <T> List<T> executeBulkIteration(BulkIterationBase<?> iteration) throws Exception {
	Operator<?> inputOp = iteration.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The iteration " + iteration.getName() + " has no input (initial partial solution).");
	}
	if (iteration.getNextPartialSolution() == null) {
		throw new InvalidProgramException("The iteration " + iteration.getName() + " has no next partial solution defined (is not closed).");
	}
	
	List<T> inputData = (List<T>) execute(inputOp);
	
	// get the operators that are iterative
	Set<Operator<?>> dynamics = new LinkedHashSet<Operator<?>>();
	DynamicPathCollector dynCollector = new DynamicPathCollector(dynamics);
	iteration.getNextPartialSolution().accept(dynCollector);
	if (iteration.getTerminationCriterion() != null) {
		iteration.getTerminationCriterion().accept(dynCollector);
	}
	
	// register the aggregators
	for (AggregatorWithName<?> a : iteration.getAggregators().getAllRegisteredAggregators()) {
		aggregators.put(a.getName(), a.getAggregator());
	}
	
	String convCriterionAggName = iteration.getAggregators().getConvergenceCriterionAggregatorName();
	ConvergenceCriterion<Value> convCriterion = (ConvergenceCriterion<Value>) iteration.getAggregators().getConvergenceCriterion();
	
	List<T> currentResult = inputData;
	
	final int maxIterations = iteration.getMaximumNumberOfIterations();
	
	for (int superstep = 1; superstep <= maxIterations; superstep++) {
		
		// set the input to the current partial solution
		this.intermediateResults.put(iteration.getPartialSolution(), currentResult);

		// set the superstep number
		iterationSuperstep = superstep;

		// grab the current iteration result
		currentResult = (List<T>) execute(iteration.getNextPartialSolution(), superstep);

		// evaluate the termination criterion
		if (iteration.getTerminationCriterion() != null) {
			execute(iteration.getTerminationCriterion(), superstep);
		}
		
		// evaluate the aggregator convergence criterion
		if (convCriterion != null && convCriterionAggName != null) {
			Value v = aggregators.get(convCriterionAggName).getAggregate();
			if (convCriterion.isConverged(superstep, v)) {
				break;
			}
		}
		
		// clear the dynamic results
		for (Operator<?> o : dynamics) {
			intermediateResults.remove(o);
		}
		
		// set the previous iteration's aggregates and reset the aggregators
		for (Map.Entry<String, Aggregator<?>> e : aggregators.entrySet()) {
			previousAggregates.put(e.getKey(), e.getValue().getAggregate());
			e.getValue().reset();
		}
	}
	
	previousAggregates.clear();
	aggregators.clear();
	
	return currentResult;
}
 

@Override
public void invoke() throws Exception {
	this.headEventReader = new MutableRecordReader<>(
			getEnvironment().getInputGate(0),
			getEnvironment().getTaskManagerInfo().getTmpDirectories());

	TaskConfig taskConfig = new TaskConfig(getTaskConfiguration());

	// store all aggregators
	this.aggregators = new HashMap<>();
	for (AggregatorWithName<?> aggWithName : taskConfig.getIterationAggregators(getUserCodeClassLoader())) {
		aggregators.put(aggWithName.getName(), aggWithName.getAggregator());
	}

	// store the aggregator convergence criterion
	if (taskConfig.usesConvergenceCriterion()) {
		convergenceCriterion = taskConfig.getConvergenceCriterion(getUserCodeClassLoader());
		convergenceAggregatorName = taskConfig.getConvergenceCriterionAggregatorName();
		Preconditions.checkNotNull(convergenceAggregatorName);
	}

	// store the default aggregator convergence criterion
	if (taskConfig.usesImplicitConvergenceCriterion()) {
		implicitConvergenceCriterion = taskConfig.getImplicitConvergenceCriterion(getUserCodeClassLoader());
		implicitConvergenceAggregatorName = taskConfig.getImplicitConvergenceCriterionAggregatorName();
		Preconditions.checkNotNull(implicitConvergenceAggregatorName);
	}

	maxNumberOfIterations = taskConfig.getNumberOfIterations();

	// set up the event handler
	int numEventsTillEndOfSuperstep = taskConfig.getNumberOfEventsUntilInterruptInIterativeGate(0);
	eventHandler = new SyncEventHandler(numEventsTillEndOfSuperstep, aggregators,
			getEnvironment().getUserClassLoader());
	headEventReader.registerTaskEventListener(eventHandler, WorkerDoneEvent.class);

	IntValue dummy = new IntValue();

	while (!terminationRequested()) {

		if (log.isInfoEnabled()) {
			log.info(formatLogString("starting iteration [" + currentIteration + "]"));
		}

		// this call listens for events until the end-of-superstep is reached
		readHeadEventChannel(dummy);

		if (log.isInfoEnabled()) {
			log.info(formatLogString("finishing iteration [" + currentIteration + "]"));
		}

		if (checkForConvergence()) {
			if (log.isInfoEnabled()) {
				log.info(formatLogString("signaling that all workers are to terminate in iteration ["
					+ currentIteration + "]"));
			}

			requestTermination();
			sendToAllWorkers(new TerminationEvent());
		} else {
			if (log.isInfoEnabled()) {
				log.info(formatLogString("signaling that all workers are done in iteration [" + currentIteration
					+ "]"));
			}

			AllWorkersDoneEvent allWorkersDoneEvent = new AllWorkersDoneEvent(aggregators);
			sendToAllWorkers(allWorkersDoneEvent);

			// reset all aggregators
			for (Aggregator<?> agg : aggregators.values()) {
				agg.reset();
			}
			currentIteration++;
		}
	}
}