Java Code Examples for org.apache.flink.api.common.operators.ResourceSpec#UNKNOWN

The following examples show how to use org.apache.flink.api.common.operators.ResourceSpec#UNKNOWN . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: Dispatcher.java From flink with Apache License 2.0

6 votes

private boolean isPartialResourceConfigured(JobGraph jobGraph) {
	boolean hasVerticesWithUnknownResource = false;
	boolean hasVerticesWithConfiguredResource = false;

	for (JobVertex jobVertex : jobGraph.getVertices()) {
		if (jobVertex.getMinResources() == ResourceSpec.UNKNOWN) {
			hasVerticesWithUnknownResource = true;
		} else {
			hasVerticesWithConfiguredResource = true;
		}

		if (hasVerticesWithUnknownResource && hasVerticesWithConfiguredResource) {
			return true;
		}
	}

	return false;
}

Example 2

Source File: Dispatcher.java From flink with Apache License 2.0

6 votes

private boolean isPartialResourceConfigured(JobGraph jobGraph) {
	boolean hasVerticesWithUnknownResource = false;
	boolean hasVerticesWithConfiguredResource = false;

	for (JobVertex jobVertex : jobGraph.getVertices()) {
		if (jobVertex.getMinResources() == ResourceSpec.UNKNOWN) {
			hasVerticesWithUnknownResource = true;
		} else {
			hasVerticesWithConfiguredResource = true;
		}

		if (hasVerticesWithUnknownResource && hasVerticesWithConfiguredResource) {
			return true;
		}
	}

	return false;
}

Example 3

Source File: StreamingJobGraphGeneratorTest.java From flink with Apache License 2.0

6 votes

@Test
public void testManagedMemoryFractionForUnknownResourceSpec() throws Exception {
	final ResourceSpec resource = ResourceSpec.UNKNOWN;
	final List<ResourceSpec> resourceSpecs = Arrays.asList(resource, resource, resource, resource);
	final List<Integer> managedMemoryWeights = Arrays.asList(1, 2, 3, 4);

	// v1(source -> map1), v2(map2) are in the same slot sharing group, v3(map3) is in a different group
	final JobGraph jobGraph = createJobGraphForManagedMemoryFractionTest(resourceSpecs, managedMemoryWeights);
	final JobVertex vertex1 = jobGraph.getVerticesSortedTopologicallyFromSources().get(0);
	final JobVertex vertex2 = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
	final JobVertex vertex3 = jobGraph.getVerticesSortedTopologicallyFromSources().get(2);

	final StreamConfig sourceConfig = new StreamConfig(vertex1.getConfiguration());
	assertEquals(1.0 / 6, sourceConfig.getManagedMemoryFraction(), 0.000001);

	final StreamConfig map1Config = Iterables.getOnlyElement(
		sourceConfig.getTransitiveChainedTaskConfigs(StreamingJobGraphGeneratorTest.class.getClassLoader()).values());
	assertEquals(2.0 / 6, map1Config.getManagedMemoryFraction(), 0.000001);

	final StreamConfig map2Config = new StreamConfig(vertex2.getConfiguration());
	assertEquals(3.0 / 6, map2Config.getManagedMemoryFraction(), 0.000001);

	final StreamConfig map3Config = new StreamConfig(vertex3.getConfiguration());
	assertEquals(1.0, map3Config.getManagedMemoryFraction(), 0.000001);

}

Example 4

Source File: StreamGraph.java From flink with Apache License 2.0

4 votes

public Tuple2<StreamNode, StreamNode> createIterationSourceAndSink(
	int loopId,
	int sourceId,
	int sinkId,
	long timeout,
	int parallelism,
	int maxParallelism,
	ResourceSpec minResources,
	ResourceSpec preferredResources) {

	final String coLocationGroup = "IterationCoLocationGroup-" + loopId;

	StreamNode source = this.addNode(sourceId,
		null,
		coLocationGroup,
		StreamIterationHead.class,
		null,
		ITERATION_SOURCE_NAME_PREFIX + "-" + loopId);
	sources.add(source.getId());
	setParallelism(source.getId(), parallelism);
	setMaxParallelism(source.getId(), maxParallelism);
	setResources(source.getId(), minResources, preferredResources);

	StreamNode sink = this.addNode(sinkId,
		null,
		coLocationGroup,
		StreamIterationTail.class,
		null,
		ITERATION_SINK_NAME_PREFIX + "-" + loopId);
	sinks.add(sink.getId());
	setParallelism(sink.getId(), parallelism);
	setMaxParallelism(sink.getId(), parallelism);
	// The tail node is always in the same slot sharing group with the head node
	// so that they can share resources (they do not use non-sharable resources,
	// i.e. managed memory). There is no contract on how the resources should be
	// divided for head and tail nodes at the moment. To be simple, we assign all
	// resources to the head node and set the tail node resources to be zero if
	// resources are specified.
	final ResourceSpec tailResources = minResources.equals(ResourceSpec.UNKNOWN)
		? ResourceSpec.UNKNOWN
		: ResourceSpec.ZERO;
	setResources(sink.getId(), tailResources, tailResources);

	iterationSourceSinkPairs.add(new Tuple2<>(source, sink));

	this.vertexIDtoBrokerID.put(source.getId(), "broker-" + loopId);
	this.vertexIDtoBrokerID.put(sink.getId(), "broker-" + loopId);
	this.vertexIDtoLoopTimeout.put(source.getId(), timeout);
	this.vertexIDtoLoopTimeout.put(sink.getId(), timeout);

	return new Tuple2<>(source, sink);
}