Java Code Examples for org.apache.flink.runtime.clusterframework.types.ResourceID#generate()

The following examples show how to use org.apache.flink.runtime.clusterframework.types.ResourceID#generate() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: YarnResourceManagerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Create mock RM dependencies.
 */
Context() throws Exception {
	rpcService = new TestingRpcService();
	rmServices = new MockResourceManagerRuntimeServices();

	// resource manager
	rmResourceID = ResourceID.generate();
	resourceManager =
			new TestingYarnResourceManager(
					rpcService,
					RM_ADDRESS,
					rmResourceID,
					flinkConfig,
					env,
					rmServices.highAvailabilityServices,
					rmServices.heartbeatServices,
					rmServices.slotManager,
					rmServices.metricRegistry,
					rmServices.jobLeaderIdService,
					new ClusterInformation("localhost", 1234),
					testingFatalErrorHandler,
					null,
					mockResourceManagerClient,
					mockNMClient,
					mockJMMetricGroup);
}
 
Example 2
Source File: TaskManagerLocationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetHostname2() {
	try {
		final String addressString = "192.168.254.254";

		// we mock the addresses to save the times of the reverse name lookups
		InetAddress address = mock(InetAddress.class);
		when(address.getCanonicalHostName()).thenReturn("192.168.254.254");
		when(address.getHostName()).thenReturn("192.168.254.254");
		when(address.getHostAddress()).thenReturn("192.168.254.254");
		when(address.getAddress()).thenReturn(new byte[] {(byte) 192, (byte) 168, (byte) 254, (byte) 254} );

		TaskManagerLocation info = new TaskManagerLocation(ResourceID.generate(), address, 54152);

		assertNotNull(info.getFQDNHostname());
		assertTrue(info.getFQDNHostname().equals(addressString));

		assertNotNull(info.getHostname());
		assertTrue(info.getHostname().equals(addressString));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 3
Source File: JobMasterTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
	configuration = new Configuration();
	haServices = new TestingHighAvailabilityServices();
	jobMasterId = JobMasterId.generate();
	jmResourceId = ResourceID.generate();

	testingFatalErrorHandler = new TestingFatalErrorHandler();

	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());

	rmLeaderRetrievalService = new SettableLeaderRetrievalService(
		null,
		null);
	haServices.setResourceManagerLeaderRetriever(rmLeaderRetrievalService);

	configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
}
 
Example 4
Source File: HeartbeatManagerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the last heartbeat from an unregistered target equals -1.
 */
@Test
public void testLastHeartbeatFromUnregisteredTarget() {
	final long heartbeatTimeout = 100L;
	final ResourceID resourceId = ResourceID.generate();
	@SuppressWarnings("unchecked")
	final HeartbeatListener<Object, Object> heartbeatListener = mock(HeartbeatListener.class);

	HeartbeatManager<?, ?> heartbeatManager = new HeartbeatManagerImpl<>(
		heartbeatTimeout,
		resourceId,
		heartbeatListener,
		mock(ScheduledExecutor.class),
		LOG);

	try {
		assertEquals(-1L, heartbeatManager.getLastHeartbeatFrom(ResourceID.generate()));
	} finally {
		heartbeatManager.stop();
	}
}
 
Example 5
Source File: TaskManagerLocationTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetHostname2() {
	try {
		final String addressString = "192.168.254.254";

		// we mock the addresses to save the times of the reverse name lookups
		InetAddress address = mock(InetAddress.class);
		when(address.getCanonicalHostName()).thenReturn("192.168.254.254");
		when(address.getHostName()).thenReturn("192.168.254.254");
		when(address.getHostAddress()).thenReturn("192.168.254.254");
		when(address.getAddress()).thenReturn(new byte[] {(byte) 192, (byte) 168, (byte) 254, (byte) 254} );

		TaskManagerLocation info = new TaskManagerLocation(ResourceID.generate(), address, 54152);

		assertNotNull(info.getFQDNHostname());
		assertTrue(info.getFQDNHostname().equals(addressString));

		assertNotNull(info.getHostname());
		assertTrue(info.getHostname().equals(addressString));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 6
Source File: SimpleSlotProvider.java    From flink with Apache License 2.0 6 votes vote down vote up
public SimpleSlotProvider(int numSlots, TaskManagerGateway taskManagerGateway) {
	checkArgument(numSlots >= 0, "numSlots must be >= 0");

	this.slots = new ArrayDeque<>(numSlots);

	for (int i = 0; i < numSlots; i++) {
		SimpleSlotContext as = new SimpleSlotContext(
			new AllocationID(),
			new TaskManagerLocation(ResourceID.generate(), InetAddress.getLoopbackAddress(), 10000 + i),
			0,
			taskManagerGateway,
			ResourceProfile.ANY);
		slots.add(as);
	}

	allocatedSlots = new HashMap<>(slots.size());
}
 
Example 7
Source File: DefaultJobMasterServiceFactory.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public JobMaster createJobMasterService(JobGraph jobGraph, OnCompletionActions jobCompletionActions, ClassLoader userCodeClassloader) throws Exception {
	return new JobMaster(
		rpcService,
		jobMasterConfiguration,
		ResourceID.generate(),
		jobGraph,
		haServices,
		slotPoolFactory,
		schedulerFactory,
		jobManagerSharedServices,
		heartbeatServices,
		jobManagerJobMetricGroupFactory,
		jobCompletionActions,
		fatalErrorHandler,
		userCodeClassloader);
}
 
Example 8
Source File: TaskExecutorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testDynamicSlotAllocation() throws Exception {
	final AllocationID allocationId = new AllocationID();
	try (TaskExecutorTestingContext submissionContext = createTaskExecutorTestingContext(2)) {
		submissionContext.start();
		final CompletableFuture<Tuple3<ResourceID, InstanceID, SlotReport>> initialSlotReportFuture =
			new CompletableFuture<>();
		ResourceManagerId resourceManagerId = createAndRegisterResourceManager(initialSlotReportFuture);
		initialSlotReportFuture.get();
		final ResourceProfile resourceProfile = DEFAULT_RESOURCE_PROFILE
			.merge(ResourceProfile.newBuilder().setCpuCores(0.1).build());

		submissionContext.taskExecutor
			.getSelfGateway(TaskExecutorGateway.class)
			.requestSlot(
				SlotID.generateDynamicSlotID(ResourceID.generate()),
				jobId,
				allocationId,
				resourceProfile,
				submissionContext.jobMasterGateway.getAddress(),
				resourceManagerId,
				timeout)
			.get();

		ResourceID resourceId = ResourceID.generate();
		SlotReport slotReport = submissionContext.taskSlotTable.createSlotReport(resourceId);
		assertThat(slotReport, containsInAnyOrder(
			new SlotStatus(new SlotID(resourceId, 0), DEFAULT_RESOURCE_PROFILE),
			new SlotStatus(new SlotID(resourceId, 1), DEFAULT_RESOURCE_PROFILE),
			new SlotStatus(SlotID.generateDynamicSlotID(resourceId), resourceProfile, jobId, allocationId)));
	}
}
 
Example 9
Source File: SlotManagerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Checks that un-registering a pending slot request will cancel it, removing it from all
 * assigned task manager slots and then remove it from the slot manager.
 */
@Test
public void testUnregisterPendingSlotRequest() throws Exception {
	final ResourceManagerId resourceManagerId = ResourceManagerId.generate();
	final ResourceActions resourceManagerActions = mock(ResourceActions.class);
	final ResourceID resourceID = ResourceID.generate();
	final SlotID slotId = new SlotID(resourceID, 0);
	final AllocationID allocationId = new AllocationID();

	final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder()
		.setRequestSlotFunction(slotIDJobIDAllocationIDStringResourceManagerIdTuple5 -> new CompletableFuture<>())
		.createTestingTaskExecutorGateway();

	final ResourceProfile resourceProfile = new ResourceProfile(1.0, 1);
	final SlotStatus slotStatus = new SlotStatus(slotId, resourceProfile);
	final SlotReport slotReport = new SlotReport(slotStatus);

	final SlotRequest slotRequest = new SlotRequest(new JobID(), allocationId, resourceProfile, "foobar");

	final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(resourceID, taskExecutorGateway);

	try (SlotManagerImpl slotManager = createSlotManager(resourceManagerId, resourceManagerActions)) {
		slotManager.registerTaskManager(taskManagerConnection, slotReport);

		TaskManagerSlot slot = slotManager.getSlot(slotId);

		slotManager.registerSlotRequest(slotRequest);

		assertNotNull(slotManager.getSlotRequest(allocationId));

		assertTrue(slot.getState() == TaskManagerSlot.State.PENDING);

		slotManager.unregisterSlotRequest(allocationId);

		assertNull(slotManager.getSlotRequest(allocationId));

		slot = slotManager.getSlot(slotId);
		assertTrue(slot.getState() == TaskManagerSlot.State.FREE);
	}
}
 
Example 10
Source File: ResourceManagerPartitionLifecycleTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private ResourceManagerGateway createAndStartResourceManager() throws Exception {
	final SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(rpcService.getScheduledExecutor())
		.build();
	final JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		TestingUtils.infiniteTime());

	final TestingResourceManager resourceManager = new TestingResourceManager(
		rpcService,
		ResourceID.generate(),
		highAvailabilityServices,
		new HeartbeatServices(100000L, 1000000L),
		slotManager,
		ResourceManagerPartitionTrackerImpl::new,
		jobLeaderIdService,
		testingFatalErrorHandler,
		UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup());

	resourceManager.start();

	// first make the ResourceManager the leader
	resourceManagerLeaderElectionService.isLeader(ResourceManagerId.generate().toUUID()).get();

	this.resourceManager = resourceManager;

	return resourceManager.getSelfGateway(ResourceManagerGateway.class);
}
 
Example 11
Source File: InstanceTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testCancelAllSlots() {
	try {
		ResourceID resourceID = ResourceID.generate();
		HardwareDescription hardwareDescription = new HardwareDescription(4, 2L*1024*1024*1024, 1024*1024*1024, 512*1024*1024);
		InetAddress address = InetAddress.getByName("127.0.0.1");
		TaskManagerLocation connection = new TaskManagerLocation(resourceID, address, 10001);

		Instance instance = new Instance(
			new ActorTaskManagerGateway(DummyActorGateway.INSTANCE),
			connection,
			new InstanceID(),
			hardwareDescription,
			3);

		assertEquals(3, instance.getNumberOfAvailableSlots());

		SimpleSlot slot1 = instance.allocateSimpleSlot();
		SimpleSlot slot2 = instance.allocateSimpleSlot();
		SimpleSlot slot3 = instance.allocateSimpleSlot();

		instance.cancelAndReleaseAllSlots();

		assertEquals(3, instance.getNumberOfAvailableSlots());

		assertTrue(slot1.isCanceled());
		assertTrue(slot2.isCanceled());
		assertTrue(slot3.isCanceled());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 12
Source File: TaskExecutorToResourceManagerConnectionTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
private static TaskExecutorRegistrationSuccess successfulRegistration() {
	return new TaskExecutorRegistrationSuccess(
		new InstanceID(),
		ResourceID.generate(),
		new ClusterInformation("blobServerHost", 55555));
}
 
Example 13
Source File: TaskExecutorLocalStateStoresManagerTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * This tests that the creation of {@link TaskManagerServices} correctly creates the local state root directory
 * for the {@link TaskExecutorLocalStateStoresManager} with the configured root directory.
 */
@Test
public void testCreationFromConfig() throws Exception {

	final Configuration config = new Configuration();

	File newFolder = temporaryFolder.newFolder();
	String tmpDir = newFolder.getAbsolutePath() + File.separator;
	final String rootDirString = "__localStateRoot1,__localStateRoot2,__localStateRoot3".replaceAll("__", tmpDir);

	// test configuration of the local state directories
	config.setString(CheckpointingOptions.LOCAL_RECOVERY_TASK_MANAGER_STATE_ROOT_DIRS, rootDirString);

	// test configuration of the local state mode
	config.setBoolean(CheckpointingOptions.LOCAL_RECOVERY, true);

	final ResourceID tmResourceID = ResourceID.generate();

	TaskManagerServicesConfiguration taskManagerServicesConfiguration =
		TaskManagerServicesConfiguration.fromConfiguration(config, InetAddress.getLocalHost(), true);

	TaskManagerServices taskManagerServices = TaskManagerServices.fromConfiguration(
		taskManagerServicesConfiguration,
		tmResourceID,
		Executors.directExecutor(),
		MEM_SIZE_PARAM,
		MEM_SIZE_PARAM);

	TaskExecutorLocalStateStoresManager taskStateManager = taskManagerServices.getTaskManagerStateStore();

	// verify configured directories for local state
	String[] split = rootDirString.split(",");
	File[] rootDirectories = taskStateManager.getLocalStateRootDirectories();
	for (int i = 0; i < split.length; ++i) {
		Assert.assertEquals(
			new File(split[i], TaskManagerServices.LOCAL_STATE_SUB_DIRECTORY_ROOT),
			rootDirectories[i]);
	}

	// verify local recovery mode
	Assert.assertTrue(taskStateManager.isLocalRecoveryEnabled());

	Assert.assertEquals("localState", TaskManagerServices.LOCAL_STATE_SUB_DIRECTORY_ROOT);
	for (File rootDirectory : rootDirectories) {
		FileUtils.deleteFileOrDirectory(rootDirectory);
	}
}
 
Example 14
Source File: TaskExecutorToResourceManagerConnectionTest.java    From flink with Apache License 2.0 4 votes vote down vote up
private static TaskExecutorRegistrationSuccess successfulRegistration() {
	return new TaskExecutorRegistrationSuccess(
		new InstanceID(),
		ResourceID.generate(),
		new ClusterInformation("blobServerHost", 55555));
}
 
Example 15
Source File: SingleInputGateTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testBackwardsEventWithUninitializedChannel() throws Exception {
	// Setup environment
	TestingTaskEventPublisher taskEventPublisher = new TestingTaskEventPublisher();

	TestingResultPartitionManager partitionManager = new TestingResultPartitionManager(new NoOpResultSubpartitionView());

	// Setup reader with one local and one unknown input channel

	NettyShuffleEnvironment environment = createNettyShuffleEnvironment();
	final SingleInputGate inputGate = createInputGate(environment, 2, ResultPartitionType.PIPELINED);
	final InputChannel[] inputChannels = new InputChannel[2];
	try {
		// Local
		ResultPartitionID localPartitionId = new ResultPartitionID();

		inputChannels[0] = InputChannelBuilder.newBuilder()
			.setPartitionId(localPartitionId)
			.setPartitionManager(partitionManager)
			.setTaskEventPublisher(taskEventPublisher)
			.buildLocalChannel(inputGate);

		// Unknown
		ResultPartitionID unknownPartitionId = new ResultPartitionID();

		inputChannels[1] = InputChannelBuilder.newBuilder()
			.setChannelIndex(1)
			.setPartitionId(unknownPartitionId)
			.setPartitionManager(partitionManager)
			.setTaskEventPublisher(taskEventPublisher)
			.buildUnknownChannel(inputGate);

		setupInputGate(inputGate, inputChannels);

		// Only the local channel can request
		assertEquals(1, partitionManager.counter);

		// Send event backwards and initialize unknown channel afterwards
		final TaskEvent event = new TestTaskEvent();
		inputGate.sendTaskEvent(event);

		// Only the local channel can send out the event
		assertEquals(1, taskEventPublisher.counter);

		// After the update, the pending event should be send to local channel

		ResourceID location = ResourceID.generate();
		inputGate.updateInputChannel(location, createRemoteWithIdAndLocation(unknownPartitionId.getPartitionId(), location));

		assertEquals(2, partitionManager.counter);
		assertEquals(2, taskEventPublisher.counter);
	}
	finally {
		inputGate.close();
		environment.close();
	}
}
 
Example 16
Source File: SlotManagerImplTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests that free slots which are reported as allocated won't be considered for fulfilling
 * other pending slot requests.
 *
 * <p>See: FLINK-8505
 */
@Test
public void testReportAllocatedSlot() throws Exception {
	final ResourceID taskManagerId = ResourceID.generate();
	final ResourceActions resourceActions = new TestingResourceActionsBuilder().build();
	final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().createTestingTaskExecutorGateway();
	final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(taskManagerId, taskExecutorGateway);

	try (final SlotManagerImpl slotManager = createSlotManager(ResourceManagerId.generate(), resourceActions)) {

		// initially report a single slot as free
		final SlotID slotId = new SlotID(taskManagerId, 0);
		final SlotStatus initialSlotStatus = new SlotStatus(
			slotId,
			ResourceProfile.ANY);
		final SlotReport initialSlotReport = new SlotReport(initialSlotStatus);

		slotManager.registerTaskManager(taskExecutorConnection, initialSlotReport);

		assertThat(slotManager.getNumberRegisteredSlots(), is(equalTo(1)));

		// Now report this slot as allocated
		final SlotStatus slotStatus = new SlotStatus(
			slotId,
			ResourceProfile.ANY,
			new JobID(),
			new AllocationID());
		final SlotReport slotReport = new SlotReport(
			slotStatus);

		slotManager.reportSlotStatus(
			taskExecutorConnection.getInstanceID(),
			slotReport);

		// this slot request should not be fulfilled
		final AllocationID allocationId = new AllocationID();
		final SlotRequest slotRequest = new SlotRequest(
			new JobID(),
			allocationId,
			ResourceProfile.UNKNOWN,
			"foobar");

		// This triggered an IllegalStateException before
		slotManager.registerSlotRequest(slotRequest);

		assertThat(slotManager.getSlotRequest(allocationId).isAssigned(), is(false));
	}
}
 
Example 17
Source File: SingleInputGateTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests that input gate can successfully convert unknown input channels into local and remote
 * channels.
 */
@Test
public void testUpdateUnknownInputChannel() throws Exception {
	final NettyShuffleEnvironment network = createNettyShuffleEnvironment();

	final ResultPartition localResultPartition = new ResultPartitionBuilder()
		.setResultPartitionManager(network.getResultPartitionManager())
		.setupBufferPoolFactoryFromNettyShuffleEnvironment(network)
		.build();

	final ResultPartition remoteResultPartition = new ResultPartitionBuilder()
		.setResultPartitionManager(network.getResultPartitionManager())
		.setupBufferPoolFactoryFromNettyShuffleEnvironment(network)
		.build();

	localResultPartition.setup();
	remoteResultPartition.setup();

	final SingleInputGate inputGate = createInputGate(network, 2, ResultPartitionType.PIPELINED);

	try {
		final ResultPartitionID localResultPartitionId = localResultPartition.getPartitionId();
		addUnknownInputChannel(network, inputGate, localResultPartitionId, 0);

		final ResultPartitionID remoteResultPartitionId = remoteResultPartition.getPartitionId();
		addUnknownInputChannel(network, inputGate, remoteResultPartitionId, 1);

		inputGate.setup();

		assertThat(inputGate.getInputChannels().get(remoteResultPartitionId.getPartitionId()),
			is(instanceOf((UnknownInputChannel.class))));
		assertThat(inputGate.getInputChannels().get(localResultPartitionId.getPartitionId()),
			is(instanceOf((UnknownInputChannel.class))));

		ResourceID localLocation = ResourceID.generate();

		// Trigger updates to remote input channel from unknown input channel
		inputGate.updateInputChannel(
			localLocation,
			createRemoteWithIdAndLocation(remoteResultPartitionId.getPartitionId(), ResourceID.generate()));

		assertThat(inputGate.getInputChannels().get(remoteResultPartitionId.getPartitionId()),
			is(instanceOf((RemoteInputChannel.class))));
		assertThat(inputGate.getInputChannels().get(localResultPartitionId.getPartitionId()),
			is(instanceOf((UnknownInputChannel.class))));

		// Trigger updates to local input channel from unknown input channel
		inputGate.updateInputChannel(
			localLocation,
			createRemoteWithIdAndLocation(localResultPartitionId.getPartitionId(), localLocation));

		assertThat(inputGate.getInputChannels().get(remoteResultPartitionId.getPartitionId()),
			is(instanceOf((RemoteInputChannel.class))));
		assertThat(inputGate.getInputChannels().get(localResultPartitionId.getPartitionId()),
			is(instanceOf((LocalInputChannel.class))));
	} finally {
		inputGate.close();
		network.close();
	}
}
 
Example 18
Source File: TaskExecutorSubmissionTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * This tests creates two tasks. The sender sends data but fails to send the
 * state update back to the job manager.
 * the second one blocks to be canceled
 */
@Test(timeout = 10000L)
public void testCancellingDependentAndStateUpdateFails() throws Exception {
	ResourceID producerLocation = ResourceID.generate();
	NettyShuffleDescriptor sdd =
		createRemoteWithIdAndLocation(new IntermediateResultPartitionID(), producerLocation);

	TaskDeploymentDescriptor tdd1 = createSender(sdd);
	TaskDeploymentDescriptor tdd2 = createReceiver(sdd);
	ExecutionAttemptID eid1 = tdd1.getExecutionAttemptId();
	ExecutionAttemptID eid2 = tdd2.getExecutionAttemptId();

	final CompletableFuture<Void> task1RunningFuture = new CompletableFuture<>();
	final CompletableFuture<Void> task2RunningFuture = new CompletableFuture<>();
	final CompletableFuture<Void> task1FailedFuture = new CompletableFuture<>();
	final CompletableFuture<Void> task2CanceledFuture = new CompletableFuture<>();

	final JobMasterId jobMasterId = JobMasterId.generate();
	TestingJobMasterGateway testingJobMasterGateway =
		new TestingJobMasterGatewayBuilder()
		.setFencingTokenSupplier(() -> jobMasterId)
		.setUpdateTaskExecutionStateFunction(taskExecutionState -> {
			if (taskExecutionState != null && taskExecutionState.getID().equals(eid1)) {
				return FutureUtils.completedExceptionally(
					new ExecutionGraphException("The execution attempt " + eid2 + " was not found."));
			} else {
				return CompletableFuture.completedFuture(Acknowledge.get());
			}
		})
		.build();

	try (TaskSubmissionTestEnvironment env =
		new TaskSubmissionTestEnvironment.Builder(jobId)
			.setResourceID(producerLocation)
			.setSlotSize(2)
			.addTaskManagerActionListener(eid1, ExecutionState.RUNNING, task1RunningFuture)
			.addTaskManagerActionListener(eid2, ExecutionState.RUNNING, task2RunningFuture)
			.addTaskManagerActionListener(eid1, ExecutionState.FAILED, task1FailedFuture)
			.addTaskManagerActionListener(eid2, ExecutionState.CANCELED, task2CanceledFuture)
			.setJobMasterId(jobMasterId)
			.setJobMasterGateway(testingJobMasterGateway)
			.useRealNonMockShuffleEnvironment()
			.build()) {
		TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
		TaskSlotTable taskSlotTable = env.getTaskSlotTable();

		taskSlotTable.allocateSlot(0, jobId, tdd1.getAllocationId(), Time.seconds(60));
		tmGateway.submitTask(tdd1, jobMasterId, timeout).get();
		task1RunningFuture.get();

		taskSlotTable.allocateSlot(1, jobId, tdd2.getAllocationId(), Time.seconds(60));
		tmGateway.submitTask(tdd2, jobMasterId, timeout).get();
		task2RunningFuture.get();

		task1FailedFuture.get();
		assertSame(taskSlotTable.getTask(eid1).getExecutionState(), ExecutionState.FAILED);

		tmGateway.cancelTask(eid2, timeout);

		task2CanceledFuture.get();
		assertSame(taskSlotTable.getTask(eid2).getExecutionState(), ExecutionState.CANCELED);
	}
}
 
Example 19
Source File: SlotManagerImplTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests that pending slot requests are tried to be fulfilled upon new slot registrations.
 */
@Test
public void testFulfillingPendingSlotRequest() throws Exception {
	final ResourceManagerId resourceManagerId = ResourceManagerId.generate();
	final ResourceID resourceID = ResourceID.generate();
	final JobID jobId = new JobID();
	final SlotID slotId = new SlotID(resourceID, 0);
	final String targetAddress = "localhost";
	final AllocationID allocationId = new AllocationID();
	final ResourceProfile resourceProfile = ResourceProfile.fromResources(42.0, 1337);
	final SlotRequest slotRequest = new SlotRequest(
		jobId,
		allocationId,
		resourceProfile,
		targetAddress);

	final AtomicInteger numberAllocateResourceCalls = new AtomicInteger(0);
	ResourceActions resourceManagerActions = new TestingResourceActionsBuilder()
		.setAllocateResourceConsumer(ignored -> numberAllocateResourceCalls.incrementAndGet())
		.build();

	final CompletableFuture<Tuple6<SlotID, JobID, AllocationID, ResourceProfile, String, ResourceManagerId>> requestFuture = new CompletableFuture<>();
	// accept an incoming slot request
	final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder()
		.setRequestSlotFunction(tuple6 -> {
			requestFuture.complete(Tuple6.of(tuple6.f0, tuple6.f1, tuple6.f2, tuple6.f3, tuple6.f4, tuple6.f5));
			return CompletableFuture.completedFuture(Acknowledge.get());
		})
		.createTestingTaskExecutorGateway();

	final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(resourceID, taskExecutorGateway);

	final SlotStatus slotStatus = new SlotStatus(slotId, resourceProfile);
	final SlotReport slotReport = new SlotReport(slotStatus);

	try (SlotManagerImpl slotManager = createSlotManager(resourceManagerId, resourceManagerActions)) {

		assertTrue("The slot request should be accepted", slotManager.registerSlotRequest(slotRequest));

		assertThat(numberAllocateResourceCalls.get(), is(1));

		slotManager.registerTaskManager(
			taskExecutorConnection,
			slotReport);

		assertThat(requestFuture.get(), is(equalTo(Tuple6.of(slotId, jobId, allocationId, resourceProfile, targetAddress, resourceManagerId))));

		TaskManagerSlot slot = slotManager.getSlot(slotId);

		assertEquals("The slot has not been allocated to the expected allocation id.", allocationId, slot.getAllocationId());
	}
}
 
Example 20
Source File: LocalUnresolvedTaskManagerLocation.java    From flink with Apache License 2.0 4 votes vote down vote up
public LocalUnresolvedTaskManagerLocation() {
	super(ResourceID.generate(), "localhost", 42);
}