org.apache.flink.api.common.operators.base.DeltaIterationBase Java Examples

@Override
public boolean preVisit(Operator<?> visitable) {
	if (this.seen.add(visitable)) {
		// add to  the map
		final String name = visitable.getName();
		List<Operator<?>> list = this.map.get(name);
		if (list == null) {
			list = new ArrayList<Operator<?>>(2);
			this.map.put(name, list);
		}
		list.add(visitable);
		
		// recurse into bulk iterations
		if (visitable instanceof BulkIterationBase) {
			((BulkIterationBase) visitable).getNextPartialSolution().accept(this);
		} else if (visitable instanceof DeltaIterationBase) {
			((DeltaIterationBase) visitable).getSolutionSetDelta().accept(this);
			((DeltaIterationBase) visitable).getNextWorkset().accept(this);
		}
		
		return true;
	} else {
		return false;
	}
}
 

/**
 * Creates a new node with a single input for the optimizer plan.
 * 
 * @param iteration The iteration operator that the node represents.
 */
public WorksetIterationNode(DeltaIterationBase<?, ?> iteration) {
	super(iteration);
	
	final int[] ssKeys = iteration.getSolutionSetKeyFields();
	if (ssKeys == null || ssKeys.length == 0) {
		throw new CompilerException("Invalid WorksetIteration: No key fields defined for the solution set.");
	}
	this.solutionSetKeyFields = new FieldList(ssKeys);
	this.partitionedProperties = new GlobalProperties();
	this.partitionedProperties.setHashPartitioned(this.solutionSetKeyFields);
	
	int weight = iteration.getMaximumNumberOfIterations() > 0 ? 
		iteration.getMaximumNumberOfIterations() : DEFAULT_COST_WEIGHT;
		
	if (weight > OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT) {
		weight = OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT;
	}
	this.costWeight = weight; 
	
	this.dataProperties = Collections.<OperatorDescriptorDual>singletonList(new WorksetOpDescriptor(this.solutionSetKeyFields));
}
 

/**
 * Creates a new node with a single input for the optimizer plan.
 * 
 * @param iteration The iteration operator that the node represents.
 */
public WorksetIterationNode(DeltaIterationBase<?, ?> iteration) {
	super(iteration);
	
	final int[] ssKeys = iteration.getSolutionSetKeyFields();
	if (ssKeys == null || ssKeys.length == 0) {
		throw new CompilerException("Invalid WorksetIteration: No key fields defined for the solution set.");
	}
	this.solutionSetKeyFields = new FieldList(ssKeys);
	this.partitionedProperties = new GlobalProperties();
	this.partitionedProperties.setHashPartitioned(this.solutionSetKeyFields);
	
	int weight = iteration.getMaximumNumberOfIterations() > 0 ? 
		iteration.getMaximumNumberOfIterations() : DEFAULT_COST_WEIGHT;
		
	if (weight > OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT) {
		weight = OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT;
	}
	this.costWeight = weight; 
	
	this.dataProperties = Collections.<OperatorDescriptorDual>singletonList(new WorksetOpDescriptor(this.solutionSetKeyFields));
}
 

@Override
public boolean preVisit(Operator<?> visitable) {
	if (this.seen.add(visitable)) {
		// add to  the map
		final String name = visitable.getName();
		List<Operator<?>> list = this.map.get(name);
		if (list == null) {
			list = new ArrayList<Operator<?>>(2);
			this.map.put(name, list);
		}
		list.add(visitable);
		
		// recurse into bulk iterations
		if (visitable instanceof BulkIterationBase) {
			((BulkIterationBase) visitable).getNextPartialSolution().accept(this);
		} else if (visitable instanceof DeltaIterationBase) {
			((DeltaIterationBase) visitable).getSolutionSetDelta().accept(this);
			((DeltaIterationBase) visitable).getNextWorkset().accept(this);
		}
		
		return true;
	} else {
		return false;
	}
}
 

/**
 * Creates a new node with a single input for the optimizer plan.
 * 
 * @param iteration The iteration operator that the node represents.
 */
public WorksetIterationNode(DeltaIterationBase<?, ?> iteration) {
	super(iteration);
	
	final int[] ssKeys = iteration.getSolutionSetKeyFields();
	if (ssKeys == null || ssKeys.length == 0) {
		throw new CompilerException("Invalid WorksetIteration: No key fields defined for the solution set.");
	}
	this.solutionSetKeyFields = new FieldList(ssKeys);
	this.partitionedProperties = new GlobalProperties();
	this.partitionedProperties.setHashPartitioned(this.solutionSetKeyFields);
	
	int weight = iteration.getMaximumNumberOfIterations() > 0 ? 
		iteration.getMaximumNumberOfIterations() : DEFAULT_COST_WEIGHT;
		
	if (weight > OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT) {
		weight = OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT;
	}
	this.costWeight = weight; 
	
	this.dataProperties = Collections.<OperatorDescriptorDual>singletonList(new WorksetOpDescriptor(this.solutionSetKeyFields));
}
 

@Override
public boolean preVisit(Operator<?> visitable) {
	if (this.seen.add(visitable)) {
		// add to  the map
		final String name = visitable.getName();
		List<Operator<?>> list = this.map.get(name);
		if (list == null) {
			list = new ArrayList<Operator<?>>(2);
			this.map.put(name, list);
		}
		list.add(visitable);
		
		// recurse into bulk iterations
		if (visitable instanceof BulkIterationBase) {
			((BulkIterationBase) visitable).getNextPartialSolution().accept(this);
		} else if (visitable instanceof DeltaIterationBase) {
			((DeltaIterationBase) visitable).getSolutionSetDelta().accept(this);
			((DeltaIterationBase) visitable).getNextWorkset().accept(this);
		}
		
		return true;
	} else {
		return false;
	}
}
 

private List<?> execute(Operator<?> operator, int superStep) throws Exception {
	List<?> result = this.intermediateResults.get(operator);
	
	// if it has already been computed, use the cached variant
	if (result != null) {
		return result;
	}
	
	if (operator instanceof BulkIterationBase) {
		result = executeBulkIteration((BulkIterationBase<?>) operator);
	}
	else if (operator instanceof DeltaIterationBase) {
		result = executeDeltaIteration((DeltaIterationBase<?, ?>) operator);
	}
	else if (operator instanceof SingleInputOperator) {
		result = executeUnaryOperator((SingleInputOperator<?, ?, ?>) operator, superStep);
	}
	else if (operator instanceof DualInputOperator) {
		result = executeBinaryOperator((DualInputOperator<?, ?, ?, ?>) operator, superStep);
	}
	else if (operator instanceof GenericDataSourceBase) {
		result = executeDataSource((GenericDataSourceBase<?, ?>) operator, superStep);
	}
	else if (operator instanceof GenericDataSinkBase) {
		executeDataSink((GenericDataSinkBase<?>) operator, superStep);
		result = Collections.emptyList();
	}
	else {
		throw new RuntimeException("Cannot execute operator " + operator.getClass().getName());
	}
	
	this.intermediateResults.put(operator, result);
	
	return result;
}
 

@Override
public boolean preVisit(Operator<?> visitable) {
	if (visitable instanceof DeltaIterationBase.WorksetPlaceHolder) {
		foundWorkset = true;
	}

	return (!foundWorkset) && seenBefore.add(visitable);
}
 

private List<?> execute(Operator<?> operator, int superStep) throws Exception {
	List<?> result = this.intermediateResults.get(operator);
	
	// if it has already been computed, use the cached variant
	if (result != null) {
		return result;
	}
	
	if (operator instanceof BulkIterationBase) {
		result = executeBulkIteration((BulkIterationBase<?>) operator);
	}
	else if (operator instanceof DeltaIterationBase) {
		result = executeDeltaIteration((DeltaIterationBase<?, ?>) operator);
	}
	else if (operator instanceof SingleInputOperator) {
		result = executeUnaryOperator((SingleInputOperator<?, ?, ?>) operator, superStep);
	}
	else if (operator instanceof DualInputOperator) {
		result = executeBinaryOperator((DualInputOperator<?, ?, ?, ?>) operator, superStep);
	}
	else if (operator instanceof GenericDataSourceBase) {
		result = executeDataSource((GenericDataSourceBase<?, ?>) operator, superStep);
	}
	else if (operator instanceof GenericDataSinkBase) {
		executeDataSink((GenericDataSinkBase<?>) operator, superStep);
		result = Collections.emptyList();
	}
	else {
		throw new RuntimeException("Cannot execute operator " + operator.getClass().getName());
	}
	
	this.intermediateResults.put(operator, result);
	
	return result;
}
 

private <D, W> DeltaIterationBase<D, W> translateDeltaIteration(DeltaIterationResultSet<?, ?> untypedIterationEnd) {
	@SuppressWarnings("unchecked")
	DeltaIterationResultSet<D, W> iterationEnd = (DeltaIterationResultSet<D, W>) untypedIterationEnd;
	DeltaIteration<D, W> iterationHead = iterationEnd.getIterationHead();

	String name = iterationHead.getName() == null ? "Unnamed Delta Iteration" : iterationHead.getName();

	DeltaIterationBase<D, W> iterationOperator = new DeltaIterationBase<>(new BinaryOperatorInformation<>(iterationEnd.getType(), iterationEnd.getWorksetType(), iterationEnd.getType()),
			iterationEnd.getKeyPositions(), name);

	iterationOperator.setMaximumNumberOfIterations(iterationEnd.getMaxIterations());

	if (iterationHead.getParallelism() > 0) {
		iterationOperator.setParallelism(iterationHead.getParallelism());
	}

	DeltaIteration.SolutionSetPlaceHolder<D> solutionSetPlaceHolder = iterationHead.getSolutionSet();
	DeltaIteration.WorksetPlaceHolder<W> worksetPlaceHolder = iterationHead.getWorkset();

	translated.put(solutionSetPlaceHolder, iterationOperator.getSolutionSet());
	translated.put(worksetPlaceHolder, iterationOperator.getWorkset());

	Operator<D> translatedSolutionSet = translate(iterationEnd.getNextSolutionSet());
	Operator<W> translatedWorkset = translate(iterationEnd.getNextWorkset());

	iterationOperator.setNextWorkset(translatedWorkset);
	iterationOperator.setSolutionSetDelta(translatedSolutionSet);

	iterationOperator.setInitialSolutionSet(translate(iterationHead.getInitialSolutionSet()));
	iterationOperator.setInitialWorkset(translate(iterationHead.getInitialWorkset()));

	// register all aggregators
	iterationOperator.getAggregators().addAll(iterationHead.getAggregators());

	iterationOperator.setSolutionSetUnManaged(iterationHead.isSolutionSetUnManaged());

	return iterationOperator;
}
 

@Override
public boolean preVisit(Operator<?> visitable) {
	if (visitable instanceof DeltaIterationBase.WorksetPlaceHolder) {
		foundWorkset = true;
	}

	return (!foundWorkset) && seenBefore.add(visitable);
}
 

private List<?> execute(Operator<?> operator, int superStep) throws Exception {
	List<?> result = this.intermediateResults.get(operator);
	
	// if it has already been computed, use the cached variant
	if (result != null) {
		return result;
	}
	
	if (operator instanceof BulkIterationBase) {
		result = executeBulkIteration((BulkIterationBase<?>) operator);
	}
	else if (operator instanceof DeltaIterationBase) {
		result = executeDeltaIteration((DeltaIterationBase<?, ?>) operator);
	}
	else if (operator instanceof SingleInputOperator) {
		result = executeUnaryOperator((SingleInputOperator<?, ?, ?>) operator, superStep);
	}
	else if (operator instanceof DualInputOperator) {
		result = executeBinaryOperator((DualInputOperator<?, ?, ?, ?>) operator, superStep);
	}
	else if (operator instanceof GenericDataSourceBase) {
		result = executeDataSource((GenericDataSourceBase<?, ?>) operator, superStep);
	}
	else if (operator instanceof GenericDataSinkBase) {
		executeDataSink((GenericDataSinkBase<?>) operator, superStep);
		result = Collections.emptyList();
	}
	else {
		throw new RuntimeException("Cannot execute operator " + operator.getClass().getName());
	}
	
	this.intermediateResults.put(operator, result);
	
	return result;
}
 

private <D, W> DeltaIterationBase<D, W> translateDeltaIteration(DeltaIterationResultSet<?, ?> untypedIterationEnd) {
	@SuppressWarnings("unchecked")
	DeltaIterationResultSet<D, W> iterationEnd = (DeltaIterationResultSet<D, W>) untypedIterationEnd;
	DeltaIteration<D, W> iterationHead = iterationEnd.getIterationHead();

	String name = iterationHead.getName() == null ? "Unnamed Delta Iteration" : iterationHead.getName();

	DeltaIterationBase<D, W> iterationOperator = new DeltaIterationBase<>(new BinaryOperatorInformation<>(iterationEnd.getType(), iterationEnd.getWorksetType(), iterationEnd.getType()),
			iterationEnd.getKeyPositions(), name);

	iterationOperator.setMaximumNumberOfIterations(iterationEnd.getMaxIterations());

	if (iterationHead.getParallelism() > 0) {
		iterationOperator.setParallelism(iterationHead.getParallelism());
	}

	DeltaIteration.SolutionSetPlaceHolder<D> solutionSetPlaceHolder = iterationHead.getSolutionSet();
	DeltaIteration.WorksetPlaceHolder<W> worksetPlaceHolder = iterationHead.getWorkset();

	translated.put(solutionSetPlaceHolder, iterationOperator.getSolutionSet());
	translated.put(worksetPlaceHolder, iterationOperator.getWorkset());

	Operator<D> translatedSolutionSet = translate(iterationEnd.getNextSolutionSet());
	Operator<W> translatedWorkset = translate(iterationEnd.getNextWorkset());

	iterationOperator.setNextWorkset(translatedWorkset);
	iterationOperator.setSolutionSetDelta(translatedSolutionSet);

	iterationOperator.setInitialSolutionSet(translate(iterationHead.getInitialSolutionSet()));
	iterationOperator.setInitialWorkset(translate(iterationHead.getInitialWorkset()));

	// register all aggregators
	iterationOperator.getAggregators().addAll(iterationHead.getAggregators());

	iterationOperator.setSolutionSetUnManaged(iterationHead.isSolutionSetUnManaged());

	return iterationOperator;
}
 

@Override
public boolean preVisit(Operator<?> visitable) {
	if (visitable instanceof DeltaIterationBase.WorksetPlaceHolder) {
		foundWorkset = true;
	}

	return (!foundWorkset) && seenBefore.add(visitable);
}
 

private <D, W> DeltaIterationBase<D, W> translateDeltaIteration(DeltaIterationResultSet<?, ?> untypedIterationEnd) {
	@SuppressWarnings("unchecked")
	DeltaIterationResultSet<D, W> iterationEnd = (DeltaIterationResultSet<D, W>) untypedIterationEnd;
	DeltaIteration<D, W> iterationHead = iterationEnd.getIterationHead();

	String name = iterationHead.getName() == null ? "Unnamed Delta Iteration" : iterationHead.getName();

	DeltaIterationBase<D, W> iterationOperator = new DeltaIterationBase<>(new BinaryOperatorInformation<>(iterationEnd.getType(), iterationEnd.getWorksetType(), iterationEnd.getType()),
			iterationEnd.getKeyPositions(), name);

	iterationOperator.setMaximumNumberOfIterations(iterationEnd.getMaxIterations());

	if (iterationHead.getParallelism() > 0) {
		iterationOperator.setParallelism(iterationHead.getParallelism());
	}

	DeltaIteration.SolutionSetPlaceHolder<D> solutionSetPlaceHolder = iterationHead.getSolutionSet();
	DeltaIteration.WorksetPlaceHolder<W> worksetPlaceHolder = iterationHead.getWorkset();

	translated.put(solutionSetPlaceHolder, iterationOperator.getSolutionSet());
	translated.put(worksetPlaceHolder, iterationOperator.getWorkset());

	Operator<D> translatedSolutionSet = translate(iterationEnd.getNextSolutionSet());
	Operator<W> translatedWorkset = translate(iterationEnd.getNextWorkset());

	iterationOperator.setNextWorkset(translatedWorkset);
	iterationOperator.setSolutionSetDelta(translatedSolutionSet);

	iterationOperator.setInitialSolutionSet(translate(iterationHead.getInitialSolutionSet()));
	iterationOperator.setInitialWorkset(translate(iterationHead.getInitialWorkset()));

	// register all aggregators
	iterationOperator.getAggregators().addAll(iterationHead.getAggregators());

	iterationOperator.setSolutionSetUnManaged(iterationHead.isSolutionSetUnManaged());

	return iterationOperator;
}
 

public DeltaIterationBase<?, ?> getIterationContract() {
	return (DeltaIterationBase<?, ?>) getOperator();
}
 

public DeltaIterationBase<?, ?> getIterationContract() {
	return (DeltaIterationBase<?, ?>) getOperator();
}
 

public DeltaIterationBase<?, ?> getIterationContract() {
	return (DeltaIterationBase<?, ?>) getOperator();
}