org.apache.flink.runtime.jobgraph.IntermediateDataSet Java Examples

The following examples show how to use org.apache.flink.runtime.jobgraph.IntermediateDataSet. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DefaultLogicalResultTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
static void assertResultsEquals(
	final Iterable<IntermediateDataSet> results,
	final Iterable<DefaultLogicalResult> logicalResults) {

	final Map<IntermediateDataSetID, DefaultLogicalResult> logicalResultMap = IterableUtils
		.toStream(logicalResults)
		.collect(Collectors.toMap(DefaultLogicalResult::getId, Function.identity()));

	for (IntermediateDataSet result : results) {
		final DefaultLogicalResult logicalResult = logicalResultMap.remove(result.getId());

		assertNotNull(logicalResult);
		assertResultInfoEquals(result, logicalResult);
	}

	// this ensures the two collections exactly matches
	assertEquals(0, logicalResultMap.size());
}
Example #2
Source File: DefaultLogicalVertexTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void buildVerticesAndResults() {
	resultMap = new HashMap<>();
	producedResults = new HashSet<>();
	consumedResults = new HashSet<>();

	final int parallelism = 3;
	jobVertex = createNoOpVertex(parallelism);

	for (int i = 0; i < 5; i++) {
		final IntermediateDataSet producedDataSet = jobVertex.createAndAddResultDataSet(BLOCKING);
		producedResults.add(producedDataSet);
		resultMap.put(producedDataSet.getId(), producedDataSet);
	}

	final JobVertex upStreamJobVertex = createNoOpVertex(parallelism);
	for (int i = 0; i < 5; i++) {
		final IntermediateDataSet consumedDataSet = upStreamJobVertex.createAndAddResultDataSet(PIPELINED);
		jobVertex.connectDataSetAsInput(consumedDataSet, ALL_TO_ALL);
		consumedResults.add(consumedDataSet);
		resultMap.put(consumedDataSet.getId(), consumedDataSet);
	}
}
Example #3
Source File: DefaultLogicalResult.java    From flink with Apache License 2.0 5 votes vote down vote up 
DefaultLogicalResult(
		final IntermediateDataSet intermediateDataSet,
		final Function<JobVertexID, DefaultLogicalVertex> vertexRetriever) {

	this.intermediateDataSet = checkNotNull(intermediateDataSet);
	this.vertexRetriever = checkNotNull(vertexRetriever);
}
Example #4
Source File: DefaultLogicalTopology.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void buildVerticesAndResults(final JobGraph jobGraph) {
	final Function<JobVertexID, DefaultLogicalVertex> vertexRetriever = this::getVertex;
	final Function<IntermediateDataSetID, DefaultLogicalResult> resultRetriever = this::getResult;

	for (JobVertex jobVertex : jobGraph.getVerticesSortedTopologicallyFromSources()) {
		final DefaultLogicalVertex logicalVertex = new DefaultLogicalVertex(jobVertex, resultRetriever);
		this.verticesSorted.add(logicalVertex);
		this.idToVertexMap.put(logicalVertex.getId(), logicalVertex);

		for (IntermediateDataSet intermediateDataSet : jobVertex.getProducedDataSets()) {
			final DefaultLogicalResult logicalResult = new DefaultLogicalResult(intermediateDataSet, vertexRetriever);
			idToResultMap.put(logicalResult.getId(), logicalResult);
		}
	}
}
Example #5
Source File: DefaultLogicalVertex.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public Iterable<DefaultLogicalResult> getConsumedResults() {
	return jobVertex.getInputs().stream()
		.map(JobEdge::getSource)
		.map(IntermediateDataSet::getId)
		.map(resultRetriever)
		.collect(Collectors.toList());
}
Example #6
Source File: DefaultLogicalVertex.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public Iterable<DefaultLogicalResult> getProducedResults() {
	return jobVertex.getProducedDataSets().stream()
		.map(IntermediateDataSet::getId)
		.map(resultRetriever)
		.collect(Collectors.toList());
}
Example #7
Source File: DefaultLogicalResultTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
static void assertResultInfoEquals(
		final IntermediateDataSet result,
		final DefaultLogicalResult logicalResult) {

	assertEquals(result.getId(), logicalResult.getId());
	assertEquals(result.getResultType(), logicalResult.getResultType());
}
Example #8
Source File: DefaultLogicalTopologyTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static void assertVertexAndConnectedResultsEquals(
	final JobVertex jobVertex,
	final DefaultLogicalVertex logicalVertex) {

	assertVertexInfoEquals(jobVertex, logicalVertex);

	final List<IntermediateDataSet> consumedResults = jobVertex.getInputs().stream()
		.map(JobEdge::getSource)
		.collect(Collectors.toList());
	assertResultsEquals(consumedResults, logicalVertex.getConsumedResults());

	final List<IntermediateDataSet> producedResults = jobVertex.getProducedDataSets();
	assertResultsEquals(producedResults, logicalVertex.getProducedResults());
}