Java Code Examples for org.apache.flink.api.common.operators.Operator#setResources()

The following examples show how to use org.apache.flink.api.common.operators.Operator#setResources() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: OperatorTranslation.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
private <T> Operator<T> translate(DataSet<T> dataSet) {
	while (dataSet instanceof NoOpOperator) {
		dataSet = ((NoOpOperator<T>) dataSet).getInput();
	}

	// check if we have already translated that data set (operation or source)
	Operator<?> previous = this.translated.get(dataSet);
	if (previous != null) {

		// Union operators may only have a single output.
		// We ensure this by not reusing previously created union operators.
		// The optimizer will merge subsequent binary unions into one n-ary union.
		if (!(dataSet instanceof UnionOperator)) {
			// all other operators are reused.
			@SuppressWarnings("unchecked")
			Operator<T> typedPrevious = (Operator<T>) previous;
			return typedPrevious;
		}
	}

	Operator<T> dataFlowOp;

	if (dataSet instanceof DataSource) {
		DataSource<T> dataSource = (DataSource<T>) dataSet;
		dataFlowOp = dataSource.translateToDataFlow();
		dataFlowOp.setResources(dataSource.getMinResources(), dataSource.getPreferredResources());
	}
	else if (dataSet instanceof SingleInputOperator) {
		SingleInputOperator<?, ?, ?> singleInputOperator = (SingleInputOperator<?, ?, ?>) dataSet;
		dataFlowOp = translateSingleInputOperator(singleInputOperator);
		dataFlowOp.setResources(singleInputOperator.getMinResources(), singleInputOperator.getPreferredResources());
	}
	else if (dataSet instanceof TwoInputOperator) {
		TwoInputOperator<?, ?, ?, ?> twoInputOperator = (TwoInputOperator<?, ?, ?, ?>) dataSet;
		dataFlowOp = translateTwoInputOperator(twoInputOperator);
		dataFlowOp.setResources(twoInputOperator.getMinResources(), twoInputOperator.getPreferredResources());
	}
	else if (dataSet instanceof BulkIterationResultSet) {
		BulkIterationResultSet<?> bulkIterationResultSet = (BulkIterationResultSet<?>) dataSet;
		dataFlowOp = translateBulkIteration(bulkIterationResultSet);
		dataFlowOp.setResources(bulkIterationResultSet.getIterationHead().getMinResources(),
				bulkIterationResultSet.getIterationHead().getPreferredResources());
	}
	else if (dataSet instanceof DeltaIterationResultSet) {
		DeltaIterationResultSet<?, ?> deltaIterationResultSet = (DeltaIterationResultSet<?, ?>) dataSet;
		dataFlowOp = translateDeltaIteration(deltaIterationResultSet);
		dataFlowOp.setResources(deltaIterationResultSet.getIterationHead().getMinResources(),
				deltaIterationResultSet.getIterationHead().getPreferredResources());
	}
	else if (dataSet instanceof DeltaIteration.SolutionSetPlaceHolder || dataSet instanceof DeltaIteration.WorksetPlaceHolder) {
		throw new InvalidProgramException("A data set that is part of a delta iteration was used as a sink or action."
			+ " Did you forget to close the iteration?");
	}
	else {
		throw new RuntimeException("Error while creating the data flow plan for the program: Unknown operator or data set type: " + dataSet);
	}

	this.translated.put(dataSet, dataFlowOp);

	// take care of broadcast variables
	translateBcVariables(dataSet, dataFlowOp);

	return dataFlowOp;
}
Example 2
Source File: OperatorTranslation.java    From flink with Apache License 2.0 4 votes vote down vote up 
private <T> Operator<T> translate(DataSet<T> dataSet) {
	while (dataSet instanceof NoOpOperator) {
		dataSet = ((NoOpOperator<T>) dataSet).getInput();
	}

	// check if we have already translated that data set (operation or source)
	Operator<?> previous = this.translated.get(dataSet);
	if (previous != null) {

		// Union operators may only have a single output.
		// We ensure this by not reusing previously created union operators.
		// The optimizer will merge subsequent binary unions into one n-ary union.
		if (!(dataSet instanceof UnionOperator)) {
			// all other operators are reused.
			@SuppressWarnings("unchecked")
			Operator<T> typedPrevious = (Operator<T>) previous;
			return typedPrevious;
		}
	}

	Operator<T> dataFlowOp;

	if (dataSet instanceof DataSource) {
		DataSource<T> dataSource = (DataSource<T>) dataSet;
		dataFlowOp = dataSource.translateToDataFlow();
		dataFlowOp.setResources(dataSource.getMinResources(), dataSource.getPreferredResources());
	}
	else if (dataSet instanceof SingleInputOperator) {
		SingleInputOperator<?, ?, ?> singleInputOperator = (SingleInputOperator<?, ?, ?>) dataSet;
		dataFlowOp = translateSingleInputOperator(singleInputOperator);
		dataFlowOp.setResources(singleInputOperator.getMinResources(), singleInputOperator.getPreferredResources());
	}
	else if (dataSet instanceof TwoInputOperator) {
		TwoInputOperator<?, ?, ?, ?> twoInputOperator = (TwoInputOperator<?, ?, ?, ?>) dataSet;
		dataFlowOp = translateTwoInputOperator(twoInputOperator);
		dataFlowOp.setResources(twoInputOperator.getMinResources(), twoInputOperator.getPreferredResources());
	}
	else if (dataSet instanceof BulkIterationResultSet) {
		BulkIterationResultSet<?> bulkIterationResultSet = (BulkIterationResultSet<?>) dataSet;
		dataFlowOp = translateBulkIteration(bulkIterationResultSet);
		dataFlowOp.setResources(bulkIterationResultSet.getIterationHead().getMinResources(),
				bulkIterationResultSet.getIterationHead().getPreferredResources());
	}
	else if (dataSet instanceof DeltaIterationResultSet) {
		DeltaIterationResultSet<?, ?> deltaIterationResultSet = (DeltaIterationResultSet<?, ?>) dataSet;
		dataFlowOp = translateDeltaIteration(deltaIterationResultSet);
		dataFlowOp.setResources(deltaIterationResultSet.getIterationHead().getMinResources(),
				deltaIterationResultSet.getIterationHead().getPreferredResources());
	}
	else if (dataSet instanceof DeltaIteration.SolutionSetPlaceHolder || dataSet instanceof DeltaIteration.WorksetPlaceHolder) {
		throw new InvalidProgramException("A data set that is part of a delta iteration was used as a sink or action."
			+ " Did you forget to close the iteration?");
	}
	else {
		throw new RuntimeException("Error while creating the data flow plan for the program: Unknown operator or data set type: " + dataSet);
	}

	this.translated.put(dataSet, dataFlowOp);

	// take care of broadcast variables
	translateBcVariables(dataSet, dataFlowOp);

	return dataFlowOp;
}
Example 3
Source File: OperatorTranslation.java    From flink with Apache License 2.0 4 votes vote down vote up 
private <T> Operator<T> translate(DataSet<T> dataSet) {
	while (dataSet instanceof NoOpOperator) {
		dataSet = ((NoOpOperator<T>) dataSet).getInput();
	}

	// check if we have already translated that data set (operation or source)
	Operator<?> previous = this.translated.get(dataSet);
	if (previous != null) {

		// Union operators may only have a single output.
		// We ensure this by not reusing previously created union operators.
		// The optimizer will merge subsequent binary unions into one n-ary union.
		if (!(dataSet instanceof UnionOperator)) {
			// all other operators are reused.
			@SuppressWarnings("unchecked")
			Operator<T> typedPrevious = (Operator<T>) previous;
			return typedPrevious;
		}
	}

	Operator<T> dataFlowOp;

	if (dataSet instanceof DataSource) {
		DataSource<T> dataSource = (DataSource<T>) dataSet;
		dataFlowOp = dataSource.translateToDataFlow();
		dataFlowOp.setResources(dataSource.getMinResources(), dataSource.getPreferredResources());
	}
	else if (dataSet instanceof SingleInputOperator) {
		SingleInputOperator<?, ?, ?> singleInputOperator = (SingleInputOperator<?, ?, ?>) dataSet;
		dataFlowOp = translateSingleInputOperator(singleInputOperator);
		dataFlowOp.setResources(singleInputOperator.getMinResources(), singleInputOperator.getPreferredResources());
	}
	else if (dataSet instanceof TwoInputOperator) {
		TwoInputOperator<?, ?, ?, ?> twoInputOperator = (TwoInputOperator<?, ?, ?, ?>) dataSet;
		dataFlowOp = translateTwoInputOperator(twoInputOperator);
		dataFlowOp.setResources(twoInputOperator.getMinResources(), twoInputOperator.getPreferredResources());
	}
	else if (dataSet instanceof BulkIterationResultSet) {
		BulkIterationResultSet<?> bulkIterationResultSet = (BulkIterationResultSet<?>) dataSet;
		dataFlowOp = translateBulkIteration(bulkIterationResultSet);
		dataFlowOp.setResources(bulkIterationResultSet.getIterationHead().getMinResources(),
				bulkIterationResultSet.getIterationHead().getPreferredResources());
	}
	else if (dataSet instanceof DeltaIterationResultSet) {
		DeltaIterationResultSet<?, ?> deltaIterationResultSet = (DeltaIterationResultSet<?, ?>) dataSet;
		dataFlowOp = translateDeltaIteration(deltaIterationResultSet);
		dataFlowOp.setResources(deltaIterationResultSet.getIterationHead().getMinResources(),
				deltaIterationResultSet.getIterationHead().getPreferredResources());
	}
	else if (dataSet instanceof DeltaIteration.SolutionSetPlaceHolder || dataSet instanceof DeltaIteration.WorksetPlaceHolder) {
		throw new InvalidProgramException("A data set that is part of a delta iteration was used as a sink or action."
			+ " Did you forget to close the iteration?");
	}
	else {
		throw new RuntimeException("Error while creating the data flow plan for the program: Unknown operator or data set type: " + dataSet);
	}

	this.translated.put(dataSet, dataFlowOp);

	// take care of broadcast variables
	translateBcVariables(dataSet, dataFlowOp);

	return dataFlowOp;
}