org.apache.flink.api.common.operators.ResourceSpec Java Examples

private boolean isPartialResourceConfigured(JobGraph jobGraph) {
	boolean hasVerticesWithUnknownResource = false;
	boolean hasVerticesWithConfiguredResource = false;

	for (JobVertex jobVertex : jobGraph.getVertices()) {
		if (jobVertex.getMinResources() == ResourceSpec.UNKNOWN) {
			hasVerticesWithUnknownResource = true;
		} else {
			hasVerticesWithConfiguredResource = true;
		}

		if (hasVerticesWithUnknownResource && hasVerticesWithConfiguredResource) {
			return true;
		}
	}

	return false;
}
 

public static ResourceProfile fromResourceSpec(ResourceSpec resourceSpec, int networkMemory) {
	if (ResourceSpec.UNKNOWN.equals(resourceSpec)) {
		return UNKNOWN;
	}

	Map<String, Resource> copiedExtendedResources = new HashMap<>(resourceSpec.getExtendedResources());

	return new ResourceProfile(
			resourceSpec.getCpuCores(),
			resourceSpec.getHeapMemory(),
			resourceSpec.getDirectMemory(),
			resourceSpec.getNativeMemory(),
			networkMemory,
			resourceSpec.getManagedMemory(),
			copiedExtendedResources);
}
 

/**
 * Test iteration job, check slot sharing group and co-location group.
 */
@Test
public void testIteration() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 2, 3).name("source");
	IterativeStream<Integer> iteration = source.iterate(3000);
	iteration.name("iteration").setParallelism(2);
	DataStream<Integer> map = iteration.map(x -> x + 1).name("map").setParallelism(2);
	DataStream<Integer> filter = map.filter((x) -> false).name("filter").setParallelism(2);
	iteration.closeWith(filter).print();

	final ResourceSpec resources = ResourceSpec.newBuilder(1.0, 100).build();
	iteration.getTransformation().setResources(resources, resources);

	StreamGraph streamGraph = env.getStreamGraph();
	for (Tuple2<StreamNode, StreamNode> iterationPair : streamGraph.getIterationSourceSinkPairs()) {
		assertNotNull(iterationPair.f0.getCoLocationGroup());
		assertEquals(iterationPair.f0.getCoLocationGroup(), iterationPair.f1.getCoLocationGroup());

		assertEquals(StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, iterationPair.f0.getSlotSharingGroup());
		assertEquals(iterationPair.f0.getSlotSharingGroup(), iterationPair.f1.getSlotSharingGroup());

		final ResourceSpec sourceMinResources = iterationPair.f0.getMinResources();
		final ResourceSpec sinkMinResources = iterationPair.f1.getMinResources();
		final ResourceSpec iterationResources = sourceMinResources.merge(sinkMinResources);
		assertThat(iterationResources, equalsResourceSpec(resources));
	}
}
 

@Test
public void testSlotSharingGroupWithSpecifiedResources() {
	final ResourceSpec resource1 = ResourceSpec.newBuilder(0.1, 100).build();
	final ResourceSpec resource2 = ResourceSpec.newBuilder(0.2, 200).build();

	final JobVertex v1 = new JobVertex("v1");
	final JobVertex v2 = new JobVertex("v2");

	v1.setResources(resource1, resource1);
	v2.setResources(resource2, resource2);

	final SlotSharingGroup group1 = new SlotSharingGroup();
	final SlotSharingGroup group2 = new SlotSharingGroup();

	v1.setSlotSharingGroup(group1);
	assertEquals(resource1, group1.getResourceSpec());

	v2.setSlotSharingGroup(group1);
	assertEquals(resource1.merge(resource2), group1.getResourceSpec());

	// v1 is moved from group1 to group2
	v1.setSlotSharingGroup(group2);
	assertEquals(resource1, group2.getResourceSpec());
	assertEquals(resource2, group1.getResourceSpec());
}
 

@Test
public void testCompareTo() throws Exception {
	ResourceSpec rs1 = ResourceSpec.newBuilder().setCpuCores(1.0).setHeapMemoryInMB(100).build();
	ResourceSpec rs2 = ResourceSpec.newBuilder().setCpuCores(1.0).setHeapMemoryInMB(100).build();
	assertEquals(0, ResourceProfile.fromResourceSpec(rs1, 0).compareTo(ResourceProfile.fromResourceSpec(rs2, 0)));

	ResourceSpec rs3 = ResourceSpec.newBuilder().
			setCpuCores(1.0).
			setHeapMemoryInMB(100).
			setGPUResource(2.2).
			build();
	assertEquals(-1, ResourceProfile.fromResourceSpec(rs1,  0).compareTo(ResourceProfile.fromResourceSpec(rs3, 0)));
	assertEquals(1, ResourceProfile.fromResourceSpec(rs3, 0).compareTo(ResourceProfile.fromResourceSpec(rs1, 0)));

	ResourceSpec rs4 = ResourceSpec.newBuilder().
			setCpuCores(1.0).
			setHeapMemoryInMB(100).
			setGPUResource(1.1).
			build();
	assertEquals(1, ResourceProfile.fromResourceSpec(rs3, 0).compareTo(ResourceProfile.fromResourceSpec(rs4, 0)));
	assertEquals(-1, ResourceProfile.fromResourceSpec(rs4, 0).compareTo(ResourceProfile.fromResourceSpec(rs3, 0)));

	ResourceSpec rs5 = ResourceSpec.newBuilder().
			setCpuCores(1.0).
			setHeapMemoryInMB(100).
			setGPUResource(2.2).
			build();
	assertEquals(0, ResourceProfile.fromResourceSpec(rs3, 0).compareTo(ResourceProfile.fromResourceSpec(rs5, 0)));
}
 

/**
 * Sets the resources for this operator, the minimum and preferred resources are the same by default.
 *
 * @param resources The resources for this operator.
 * @return The operator with set minimum and preferred resources.
 */
private SingleOutputStreamOperator<T> setResources(ResourceSpec resources) {
	Preconditions.checkNotNull(resources, "The resources must be not null.");
	Preconditions.checkArgument(resources.isValid(), "The values in resources must be not less than 0.");

	transformation.setResources(resources, resources);

	return this;
}
 

/**
 * Sets the minimum and preferred resources for this operator, and the lower and upper resource limits will
 * be considered in dynamic resource resize feature for future plan.
 *
 * @param minResources The minimum resources for this operator.
 * @param preferredResources The preferred resources for this operator.
 * @return The operator with set minimum and preferred resources.
 */
private SingleOutputStreamOperator<T> setResources(ResourceSpec minResources, ResourceSpec preferredResources) {
	Preconditions.checkNotNull(minResources, "The min resources must be not null.");
	Preconditions.checkNotNull(preferredResources, "The preferred resources must be not null.");
	Preconditions.checkArgument(minResources.isValid() && preferredResources.isValid() && minResources.lessThanOrEqual(preferredResources),
			"The values in resources must be not less than 0 and the preferred resources must be greater than the min resources.");

	transformation.setResources(minResources, preferredResources);

	return this;
}
 

@Test
public void testGet() throws Exception {
	ResourceSpec rs = ResourceSpec.newBuilder().
			setCpuCores(1.0).
			setHeapMemoryInMB(100).
			setGPUResource(1.6).
			build();
	ResourceProfile rp = ResourceProfile.fromResourceSpec(rs, 50);

	assertEquals(1.0, rp.getCpuCores(), 0.000001);
	assertEquals(150, rp.getMemoryInMB());
	assertEquals(100, rp.getOperatorsMemoryInMB());
	assertEquals(1.6, rp.getExtendedResources().get(ResourceSpec.GPU_NAME).getValue(), 0.000001);
}
 

/**
 * Sets the minimum and preferred resources for the iteration. This overrides the default resources.
 * The lower and upper resource limits will be considered in dynamic resource resize feature for future plan.
 *
 * @param minResources The minimum resources for the iteration.
 * @param preferredResources The preferred resources for the iteration.
 * @return The iteration with set minimum and preferred resources.
 */
private DeltaIteration<ST, WT> setResources(ResourceSpec minResources, ResourceSpec preferredResources) {
	Preconditions.checkNotNull(minResources, "The min resources must be not null.");
	Preconditions.checkNotNull(preferredResources, "The preferred resources must be not null.");
	Preconditions.checkArgument(minResources.isValid() && preferredResources.isValid() && minResources.lessThanOrEqual(preferredResources),
			"The values in resources must be not less than 0 and the preferred resources must be greater than the min resources.");

	this.minResources = minResources;
	this.preferredResources = preferredResources;

	return this;
}
 

/**
 * Sets the resources for this data sink, and the minimum and preferred resources are the same by default.
 *
 * @param resources The resources for this data sink.
 * @return The data sink with set minimum and preferred resources.
 */
private DataSink<T> setResources(ResourceSpec resources) {
	Preconditions.checkNotNull(resources, "The resources must be not null.");
	Preconditions.checkArgument(resources.isValid(), "The values in resources must be not less than 0.");

	this.minResources = resources;
	this.preferredResources = resources;

	return this;
}
 

/**
 * Sets the minimum and preferred resources for this operator, and the lower and upper resource limits will
 * be considered in dynamic resource resize feature for future plan.
 *
 * @param minResources The minimum resources for this operator.
 * @param preferredResources The preferred resources for this operator.
 * @return The operator with set minimum and preferred resources.
 */
private SingleOutputStreamOperator<T> setResources(ResourceSpec minResources, ResourceSpec preferredResources) {
	Preconditions.checkNotNull(minResources, "The min resources must be not null.");
	Preconditions.checkNotNull(preferredResources, "The preferred resources must be not null.");
	Preconditions.checkArgument(minResources.isValid() && preferredResources.isValid() && minResources.lessThanOrEqual(preferredResources),
			"The values in resources must be not less than 0 and the preferred resources must be greater than the min resources.");

	transformation.setResources(minResources, preferredResources);

	return this;
}
 

/**
 * Sets the minimum and preferred resources for the iteration. This overrides the default resources.
 * The lower and upper resource limits will be considered in dynamic resource resize feature for future plan.
 *
 * @param minResources The minimum resources for the iteration.
 * @param preferredResources The preferred resources for the iteration.
 * @return The iteration with set minimum and preferred resources.
 */
private DeltaIteration<ST, WT> setResources(ResourceSpec minResources, ResourceSpec preferredResources) {
	OperatorValidationUtils.validateMinAndPreferredResources(minResources, preferredResources);

	this.minResources = minResources;
	this.preferredResources = preferredResources;

	return this;
}
 

/**
 * Sets the resources for the iteration, and the minimum and preferred resources are the same by default.
 *	The lower and upper resource limits will be considered in dynamic resource resize feature for future plan.
 *
 * @param resources The resources for the iteration.
 * @return The iteration with set minimum and preferred resources.
 */
private DeltaIteration<ST, WT> setResources(ResourceSpec resources) {
	Preconditions.checkNotNull(resources, "The resources must be not null.");
	Preconditions.checkArgument(resources.isValid(), "The values in resources must be not less than 0.");

	this.minResources = resources;
	this.preferredResources = resources;

	return this;
}
 

/**
 * Sets the minimum and preferred resources for this data sink. and the lower and upper resource limits
 * will be considered in resource resize feature for future plan.
 *
 * @param minResources The minimum resources for this data sink.
 * @param preferredResources The preferred resources for this data sink.
 * @return The data sink with set minimum and preferred resources.
 */
private DataSink<T> setResources(ResourceSpec minResources, ResourceSpec preferredResources) {
	Preconditions.checkNotNull(minResources, "The min resources must be not null.");
	Preconditions.checkNotNull(preferredResources, "The preferred resources must be not null.");
	Preconditions.checkArgument(minResources.isValid() && preferredResources.isValid() && minResources.lessThanOrEqual(preferredResources),
			"The values in resources must be not less than 0 and the preferred resources must be greater than the min resources.");

	this.minResources = minResources;
	this.preferredResources = preferredResources;

	return this;
}
 

private ResourceSpec createChainedPreferredResources(Integer vertexID, List<StreamEdge> chainedOutputs) {
	ResourceSpec preferredResources = streamGraph.getStreamNode(vertexID).getPreferredResources();
	for (StreamEdge chainable : chainedOutputs) {
		preferredResources = preferredResources.merge(chainedPreferredResources.get(chainable.getTargetId()));
	}
	return preferredResources;
}
 

/**
 * Sets the minimum and preferred resources for this sink, and the lower and upper resource limits will
 * be considered in resource resize feature for future plan.
 *
 * @param minResources The minimum resources for this sink.
 * @param preferredResources The preferred resources for this sink
 * @return The sink with set minimum and preferred resources.
 */
private DataStreamSink<T> setResources(ResourceSpec minResources, ResourceSpec preferredResources) {
	Preconditions.checkNotNull(minResources, "The min resources must be not null.");
	Preconditions.checkNotNull(preferredResources, "The preferred resources must be not null.");
	Preconditions.checkArgument(minResources.isValid() && preferredResources.isValid() && minResources.lessThanOrEqual(preferredResources),
			"The values in resources must be not less than 0 and the preferred resources must be greater than the min resources.");

	transformation.setResources(minResources, preferredResources);

	return this;
}
 

/**
 * Sets the minimum and preferred resources for this sink, and the lower and upper resource limits will
 * be considered in resource resize feature for future plan.
 *
 * @param minResources The minimum resources for this sink.
 * @param preferredResources The preferred resources for this sink
 * @return The sink with set minimum and preferred resources.
 */
private DataStreamSink<T> setResources(ResourceSpec minResources, ResourceSpec preferredResources) {
	Preconditions.checkNotNull(minResources, "The min resources must be not null.");
	Preconditions.checkNotNull(preferredResources, "The preferred resources must be not null.");
	Preconditions.checkArgument(minResources.isValid() && preferredResources.isValid() && minResources.lessThanOrEqual(preferredResources),
			"The values in resources must be not less than 0 and the preferred resources must be greater than the min resources.");

	transformation.setResources(minResources, preferredResources);

	return this;
}
 

@Test
public void testMatchRequirement() throws Exception {
	ResourceProfile rp1 = new ResourceProfile(1.0, 100, 100, 100, 0, 0, Collections.emptyMap());
	ResourceProfile rp2 = new ResourceProfile(1.0, 200, 200, 200, 0, 0, Collections.emptyMap());
	ResourceProfile rp3 = new ResourceProfile(2.0, 100, 100, 100, 0, 0, Collections.emptyMap());
	ResourceProfile rp4 = new ResourceProfile(2.0, 200, 200, 200, 0, 0, Collections.emptyMap());

	assertFalse(rp1.isMatching(rp2));
	assertTrue(rp2.isMatching(rp1));

	assertFalse(rp1.isMatching(rp3));
	assertTrue(rp3.isMatching(rp1));

	assertFalse(rp2.isMatching(rp3));
	assertFalse(rp3.isMatching(rp2));

	assertTrue(rp4.isMatching(rp1));
	assertTrue(rp4.isMatching(rp2));
	assertTrue(rp4.isMatching(rp3));
	assertTrue(rp4.isMatching(rp4));

	ResourceProfile rp5 = new ResourceProfile(2.0, 100, 100, 100, 100, 100, null);
	assertFalse(rp4.isMatching(rp5));

	ResourceSpec rs1 = ResourceSpec.newBuilder().
			setCpuCores(1.0).
			setHeapMemoryInMB(100).
			setGPUResource(2.2).
			build();
	ResourceSpec rs2 = ResourceSpec.newBuilder().
			setCpuCores(1.0).
			setHeapMemoryInMB(100).
			setGPUResource(1.1).
			build();

	assertFalse(rp1.isMatching(ResourceProfile.fromResourceSpec(rs1, 0)));
	assertTrue(ResourceProfile.fromResourceSpec(rs1, 0).isMatching(ResourceProfile.fromResourceSpec(rs2, 0)));
	assertFalse(ResourceProfile.fromResourceSpec(rs2, 0).isMatching(ResourceProfile.fromResourceSpec(rs1, 0)));
}
 

public static void validateResourceRequirements(
		final ResourceSpec minResources,
		final ResourceSpec preferredResources,
		final int managedMemoryWeight) {

	validateMinAndPreferredResources(minResources, preferredResources);
	Preconditions.checkArgument(
		managedMemoryWeight >= 0,
		"managed memory weight must be no less than zero, was: " + managedMemoryWeight);
	Preconditions.checkArgument(
		minResources.equals(ResourceSpec.UNKNOWN) || managedMemoryWeight == Transformation.DEFAULT_MANAGED_MEMORY_WEIGHT,
		"The resources and managed memory weight should not be specified at the same time. " +
			"resources: " + minResources + ", managed memory weight: " + managedMemoryWeight);
}
 

private ResourceSpec createChainedPreferredResources(Integer vertexID, List<StreamEdge> chainedOutputs) {
	ResourceSpec preferredResources = streamGraph.getStreamNode(vertexID).getPreferredResources();
	for (StreamEdge chainable : chainedOutputs) {
		preferredResources = preferredResources.merge(chainedPreferredResources.get(chainable.getTargetId()));
	}
	return preferredResources;
}
 

/**
 * Sets batch properties for {@link StreamGraph}.
 */
public static void setBatchProperties(StreamGraph streamGraph, TableConfig tableConfig) {
	streamGraph.getStreamNodes().forEach(
			sn -> sn.setResources(ResourceSpec.UNKNOWN, ResourceSpec.UNKNOWN));
	streamGraph.setChaining(true);
	streamGraph.setAllVerticesInSameSlotSharingGroupByDefault(false);
	streamGraph.setScheduleMode(ScheduleMode.LAZY_FROM_SOURCES_WITH_BATCH_SLOT_REQUEST);
	streamGraph.setStateBackend(null);
	if (streamGraph.getCheckpointConfig().isCheckpointingEnabled()) {
		throw new IllegalArgumentException("Checkpoint is not supported for batch jobs.");
	}
	streamGraph.setGlobalDataExchangeMode(getGlobalDataExchangeMode(tableConfig));
}
 

@Test(expected = IllegalArgumentException.class)
public void testSetResourcesFailIfManagedMemoryWeightIsSpecified() {
	transformation.setManagedMemoryWeight(123);

	final ResourceSpec resources = ResourceSpec.newBuilder(1.0, 100).build();
	transformation.setResources(resources, resources);
}
 

private ResourceSpec createChainedMinResources(Integer vertexID, List<StreamEdge> chainedOutputs) {
	ResourceSpec minResources = streamGraph.getStreamNode(vertexID).getMinResources();
	for (StreamEdge chainable : chainedOutputs) {
		minResources = minResources.merge(chainedMinResources.get(chainable.getTargetId()));
	}
	return minResources;
}
 

private ResourceSpec createChainedPreferredResources(Integer vertexID, List<StreamEdge> chainedOutputs) {
	ResourceSpec preferredResources = streamGraph.getStreamNode(vertexID).getPreferredResources();
	for (StreamEdge chainable : chainedOutputs) {
		preferredResources = preferredResources.merge(chainedPreferredResources.get(chainable.getTargetId()));
	}
	return preferredResources;
}
 

private ResourceSpec createChainedMinResources(Integer vertexID, List<StreamEdge> chainedOutputs) {
	ResourceSpec minResources = streamGraph.getStreamNode(vertexID).getMinResources();
	for (StreamEdge chainable : chainedOutputs) {
		minResources = minResources.merge(chainedMinResources.get(chainable.getTargetId()));
	}
	return minResources;
}
 

/**
 * Sets the minimum and preferred resources for the iteration. This overrides the default resources.
 * The lower and upper resource limits will be considered in dynamic resource resize feature for future plan.
 *
 * @param minResources The minimum resources for the iteration.
 * @param preferredResources The preferred resources for the iteration.
 * @return The iteration with set minimum and preferred resources.
 */
private DeltaIteration<ST, WT> setResources(ResourceSpec minResources, ResourceSpec preferredResources) {
	Preconditions.checkNotNull(minResources, "The min resources must be not null.");
	Preconditions.checkNotNull(preferredResources, "The preferred resources must be not null.");
	Preconditions.checkArgument(minResources.isValid() && preferredResources.isValid() && minResources.lessThanOrEqual(preferredResources),
			"The values in resources must be not less than 0 and the preferred resources must be greater than the min resources.");

	this.minResources = minResources;
	this.preferredResources = preferredResources;

	return this;
}
 

/**
 * Sets the resources for this data sink, and the minimum and preferred resources are the same by default.
 *
 * @param resources The resources for this data sink.
 * @return The data sink with set minimum and preferred resources.
 */
private DataSink<T> setResources(ResourceSpec resources) {
	Preconditions.checkNotNull(resources, "The resources must be not null.");
	Preconditions.checkArgument(resources.isValid(), "The values in resources must be not less than 0.");

	this.minResources = resources;
	this.preferredResources = resources;

	return this;
}
 

@Test
public void testFromSpecWithSerializationCopy() throws Exception {
	final ResourceSpec copiedSpec = CommonTestUtils.createCopySerializable(ResourceSpec.UNKNOWN);
	final ResourceProfile profile = ResourceProfile.fromResourceSpec(copiedSpec);

	assertEquals(ResourceProfile.fromResourceSpec(ResourceSpec.UNKNOWN), profile);
}
 

/**
 * Sets the resources for this operator. This overrides the default minimum and preferred resources.
 *
 * @param resources The resources for this operator.
 * @return The operator with set minimum and preferred resources.
 */
private O setResources(ResourceSpec resources) {
	Preconditions.checkNotNull(resources, "The resources must be not null.");
	Preconditions.checkArgument(resources.isValid(), "The values in resources must be not less than 0.");

	this.minResources = resources;
	this.preferredResources = resources;

	@SuppressWarnings("unchecked")
	O returnType = (O) this;
	return returnType;
}
 

/**
 * Sets the minimum and preferred resources for this operator. This overrides the default resources.
 * The lower and upper resource limits will be considered in dynamic resource resize feature for future plan.
 *
 * @param minResources The minimum resources for this operator.
 * @param preferredResources The preferred resources for this operator.
 * @return The operator with set minimum and preferred resources.
 */
private O setResources(ResourceSpec minResources, ResourceSpec preferredResources) {
	Preconditions.checkNotNull(minResources, "The min resources must be not null.");
	Preconditions.checkNotNull(preferredResources, "The preferred resources must be not null.");

	Preconditions.checkArgument(minResources.isValid() && preferredResources.isValid() && minResources.lessThanOrEqual(preferredResources),
			"The values in resources must be not less than 0 and the preferred resources must be greater than the min resources.");

	this.minResources = minResources;
	this.preferredResources = preferredResources;

	@SuppressWarnings("unchecked")
	O returnType = (O) this;
	return returnType;
}