org.apache.flink.runtime.jobgraph.JobVertexID Java Examples

The following examples show how to use org.apache.flink.runtime.jobgraph.JobVertexID. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TaskLocalStateStoreImpl.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
TaskLocalStateStoreImpl(
	@Nonnull JobID jobID,
	@Nonnull AllocationID allocationID,
	@Nonnull JobVertexID jobVertexID,
	@Nonnegative int subtaskIndex,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull Executor discardExecutor,
	@Nonnull SortedMap<Long, TaskStateSnapshot> storedTaskStateByCheckpointID,
	@Nonnull Object lock) {

	this.jobID = jobID;
	this.allocationID = allocationID;
	this.jobVertexID = jobVertexID;
	this.subtaskIndex = subtaskIndex;
	this.discardExecutor = discardExecutor;
	this.localRecoveryConfig = localRecoveryConfig;
	this.storedTaskStateByCheckpointID = storedTaskStateByCheckpointID;
	this.lock = lock;
	this.disposed = false;
}
 
Example #2
Source File: CheckpointCoordinatorTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
static TaskStateSnapshot mockSubtaskState(
	JobVertexID jobVertexID,
	int index,
	KeyGroupRange keyGroupRange) throws IOException {

	OperatorStateHandle partitionableState = generatePartitionableStateHandle(jobVertexID, index, 2, 8, false);
	KeyGroupsStateHandle partitionedKeyGroupState = generateKeyGroupState(jobVertexID, keyGroupRange, false);

	TaskStateSnapshot subtaskStates = spy(new TaskStateSnapshot());
	OperatorSubtaskState subtaskState = spy(new OperatorSubtaskState(
		partitionableState, null, partitionedKeyGroupState, null)
	);

	subtaskStates.putSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID), subtaskState);

	return subtaskStates;
}
 
Example #3
Source File: CheckpointCoordinatorTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public static OperatorStateHandle generatePartitionableStateHandle(
	JobVertexID jobVertexID,
	int index,
	int namedStates,
	int partitionsPerState,
	boolean rawState) throws IOException {

	Map<String, List<? extends Serializable>> statesListsMap = new HashMap<>(namedStates);

	for (int i = 0; i < namedStates; ++i) {
		List<Integer> testStatesLists = new ArrayList<>(partitionsPerState);
		// generate state
		int seed = jobVertexID.hashCode() * index + i * namedStates;
		if (rawState) {
			seed = (seed + 1) * 31;
		}
		Random random = new Random(seed);
		for (int j = 0; j < partitionsPerState; ++j) {
			int simulatedStateValue = random.nextInt();
			testStatesLists.add(simulatedStateValue);
		}
		statesListsMap.put("state-" + i, testStatesLists);
	}

	return generatePartitionableStateHandle(statesListsMap);
}
 
Example #4
Source File: JobVertexBackPressureHandlerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testAbsentBackPressure() throws Exception {
	final Map<String, String> pathParameters = new HashMap<>();
	pathParameters.put(JobIDPathParameter.KEY, TEST_JOB_ID_BACK_PRESSURE_STATS_ABSENT.toString());
	pathParameters.put(JobVertexIdPathParameter.KEY, new JobVertexID().toString());

	final HandlerRequest<EmptyRequestBody, JobVertexMessageParameters> request =
		new HandlerRequest<>(
			EmptyRequestBody.getInstance(),
			new JobVertexMessageParameters(), pathParameters, Collections.emptyMap());

	final CompletableFuture<JobVertexBackPressureInfo> jobVertexBackPressureInfoCompletableFuture =
		jobVertexBackPressureHandler.handleRequest(request, restfulGateway);
	final JobVertexBackPressureInfo jobVertexBackPressureInfo = jobVertexBackPressureInfoCompletableFuture.get();

	assertThat(jobVertexBackPressureInfo.getStatus(), equalTo(VertexBackPressureStatus.DEPRECATED));
}
 
Example #5
Source File: MetricGroupTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateQueryServiceMetricInfo() {
	JobID jid = new JobID();
	JobVertexID vid = new JobVertexID();
	AbstractID eid = new AbstractID();
	MetricRegistryImpl registry = new MetricRegistryImpl(defaultMetricRegistryConfiguration);
	TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id");
	TaskManagerJobMetricGroup job = new TaskManagerJobMetricGroup(registry, tm, jid, "jobname");
	TaskMetricGroup task = new TaskMetricGroup(registry, job, vid, eid, "taskName", 4, 5);
	GenericMetricGroup userGroup1 = new GenericMetricGroup(registry, task, "hello");
	GenericMetricGroup userGroup2 = new GenericMetricGroup(registry, userGroup1, "world");

	QueryScopeInfo.TaskQueryScopeInfo info1 = (QueryScopeInfo.TaskQueryScopeInfo) userGroup1.createQueryServiceMetricInfo(new DummyCharacterFilter());
	assertEquals("hello", info1.scope);
	assertEquals(jid.toString(), info1.jobID);
	assertEquals(vid.toString(), info1.vertexID);
	assertEquals(4, info1.subtaskIndex);

	QueryScopeInfo.TaskQueryScopeInfo info2 = (QueryScopeInfo.TaskQueryScopeInfo) userGroup2.createQueryServiceMetricInfo(new DummyCharacterFilter());
	assertEquals("hello.world", info2.scope);
	assertEquals(jid.toString(), info2.jobID);
	assertEquals(vid.toString(), info2.vertexID);
	assertEquals(4, info2.subtaskIndex);
}
 
Example #6
Source File: OperatorGroupTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateQueryServiceMetricInfo() {
	JobID jid = new JobID();
	JobVertexID vid = new JobVertexID();
	AbstractID eid = new AbstractID();
	OperatorID oid = new OperatorID();
	TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id");
	TaskManagerJobMetricGroup job = new TaskManagerJobMetricGroup(registry, tm, jid, "jobname");
	TaskMetricGroup task = new TaskMetricGroup(registry, job, vid, eid, "taskName", 4, 5);
	OperatorMetricGroup operator = new OperatorMetricGroup(registry, task, oid, "operator");

	QueryScopeInfo.OperatorQueryScopeInfo info = operator.createQueryServiceMetricInfo(new DummyCharacterFilter());
	assertEquals("", info.scope);
	assertEquals(jid.toString(), info.jobID);
	assertEquals(vid.toString(), info.vertexID);
	assertEquals(4, info.subtaskIndex);
	assertEquals("operator", info.operatorName);
}
 
Example #7
Source File: CheckpointCoordinatorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
public static ChainedStateHandle<OperatorStateHandle> generateChainedPartitionableStateHandle(
		JobVertexID jobVertexID,
		int index,
		int namedStates,
		int partitionsPerState,
		boolean rawState) throws IOException {

	Map<String, List<? extends Serializable>> statesListsMap = new HashMap<>(namedStates);

	for (int i = 0; i < namedStates; ++i) {
		List<Integer> testStatesLists = new ArrayList<>(partitionsPerState);
		// generate state
		int seed = jobVertexID.hashCode() * index + i * namedStates;
		if (rawState) {
			seed = (seed + 1) * 31;
		}
		Random random = new Random(seed);
		for (int j = 0; j < partitionsPerState; ++j) {
			int simulatedStateValue = random.nextInt();
			testStatesLists.add(simulatedStateValue);
		}
		statesListsMap.put("state-" + i, testStatesLists);
	}

	return ChainedStateHandle.wrapSingleHandle(generatePartitionableStateHandle(statesListsMap));
}
 
Example #8
Source File: FailedCheckpointStatsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the end to end duration of a failed checkpoint is the duration
 * until the failure.
 */
@Test
public void testEndToEndDuration() throws Exception {
	long duration = 123912931293L;
	long triggerTimestamp = 10123;
	long failureTimestamp = triggerTimestamp + duration;

	Map<JobVertexID, TaskStateStats> taskStats = new HashMap<>();
	JobVertexID jobVertexId = new JobVertexID();
	taskStats.put(jobVertexId, new TaskStateStats(jobVertexId, 1));

	FailedCheckpointStats failed = new FailedCheckpointStats(
		0,
		triggerTimestamp,
		CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION),
		1,
		taskStats,
		0,
		0,
		failureTimestamp,
		null,
		null);

	assertEquals(duration, failed.getEndToEndDuration());
}
 
Example #9
Source File: CheckpointMetadataLoadingTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests correct savepoint loading.
 */
@Test
public void testAllStateRestored() throws Exception {
	final JobID jobId = new JobID();
	final OperatorID operatorId = new OperatorID();
	final long checkpointId = Integer.MAX_VALUE + 123123L;
	final int parallelism = 128128;

	final CompletedCheckpointStorageLocation testSavepoint = createSavepointWithOperatorSubtaskState(checkpointId, operatorId, parallelism);
	final Map<JobVertexID, ExecutionJobVertex> tasks = createTasks(operatorId, parallelism, parallelism);

	final CompletedCheckpoint loaded = Checkpoints.loadAndValidateCheckpoint(jobId, tasks, testSavepoint, cl, false);

	assertEquals(jobId, loaded.getJobId());
	assertEquals(checkpointId, loaded.getCheckpointID());
}
 
Example #10
Source File: AbstractExecutionSlotAllocatorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testComputeAllPriorAllocationIds() {
	final List<AllocationID> expectAllocationIds = Arrays.asList(new AllocationID(), new AllocationID());
	final List<ExecutionVertexSchedulingRequirements> testSchedulingRequirements = Arrays.asList(
		new ExecutionVertexSchedulingRequirements.Builder().
			withExecutionVertexId(new ExecutionVertexID(new JobVertexID(), 0)).
			withPreviousAllocationId(expectAllocationIds.get(0)).
			build(),
		new ExecutionVertexSchedulingRequirements.Builder().
			withExecutionVertexId(new ExecutionVertexID(new JobVertexID(), 1)).
			withPreviousAllocationId(expectAllocationIds.get(0)).
			build(),
		new ExecutionVertexSchedulingRequirements.Builder().
			withExecutionVertexId(new ExecutionVertexID(new JobVertexID(), 2)).
			withPreviousAllocationId(expectAllocationIds.get(1)).
			build(),
		new ExecutionVertexSchedulingRequirements.Builder().
			withExecutionVertexId(new ExecutionVertexID(new JobVertexID(), 3)).
			build()
	);

	final Set<AllocationID> allPriorAllocationIds =
		AbstractExecutionSlotAllocator.computeAllPriorAllocationIds(testSchedulingRequirements);
	assertThat(allPriorAllocationIds, containsInAnyOrder(expectAllocationIds.toArray()));
}
 
Example #11
Source File: BackPressureStatsTrackerImplTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private ExecutionVertex mockExecutionVertex(
		ExecutionJobVertex jobVertex,
		int subTaskIndex) {

	Execution exec = Mockito.mock(Execution.class);
	Mockito.when(exec.getAttemptId()).thenReturn(new ExecutionAttemptID());

	JobVertexID id = jobVertex.getJobVertexId();

	ExecutionVertex vertex = Mockito.mock(ExecutionVertex.class);
	Mockito.when(vertex.getJobvertexId()).thenReturn(id);
	Mockito.when(vertex.getCurrentExecutionAttempt()).thenReturn(exec);
	Mockito.when(vertex.getParallelSubtaskIndex()).thenReturn(subTaskIndex);

	return vertex;
}
 
Example #12
Source File: CheckpointStatistics.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@JsonCreator
public PendingCheckpointStatistics(
	@JsonProperty(FIELD_NAME_ID) long id,
	@JsonProperty(FIELD_NAME_STATUS) CheckpointStatsStatus status,
	@JsonProperty(FIELD_NAME_IS_SAVEPOINT) boolean savepoint,
	@JsonProperty(FIELD_NAME_TRIGGER_TIMESTAMP) long triggerTimestamp,
	@JsonProperty(FIELD_NAME_LATEST_ACK_TIMESTAMP) long latestAckTimestamp,
	@JsonProperty(FIELD_NAME_STATE_SIZE) long stateSize,
	@JsonProperty(FIELD_NAME_DURATION) long duration,
	@JsonProperty(FIELD_NAME_ALIGNMENT_BUFFERED) long alignmentBuffered,
	@JsonProperty(FIELD_NAME_NUM_SUBTASKS) int numSubtasks,
	@JsonProperty(FIELD_NAME_NUM_ACK_SUBTASKS) int numAckSubtasks,
	@JsonDeserialize(keyUsing = JobVertexIDKeyDeserializer.class) @JsonProperty(FIELD_NAME_TASKS) Map<JobVertexID, TaskCheckpointStatistics> checkpointingStatisticsPerTask) {
	super(
		id,
		status,
		savepoint,
		triggerTimestamp,
		latestAckTimestamp,
		stateSize,
		duration,
		alignmentBuffered,
		numSubtasks,
		numAckSubtasks,
		checkpointingStatisticsPerTask);
}
 
Example #13
Source File: SchedulerTestUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
public static Execution getTestVertex(Collection<CompletableFuture<TaskManagerLocation>> preferredLocationFutures) {
	ExecutionJobVertex executionJobVertex = mock(ExecutionJobVertex.class);
	ExecutionVertex vertex = mock(ExecutionVertex.class);

	when(vertex.getPreferredLocationsBasedOnInputs()).thenReturn(preferredLocationFutures);
	when(vertex.getPreferredLocations()).thenReturn(preferredLocationFutures);
	when(vertex.getJobId()).thenReturn(new JobID());
	when(vertex.toString()).thenReturn("TEST-VERTEX");
	when(vertex.getJobVertex()).thenReturn(executionJobVertex);
	when(vertex.getJobvertexId()).thenReturn(new JobVertexID());

	Execution execution = mock(Execution.class);
	when(execution.getVertex()).thenReturn(vertex);
	when(execution.calculatePreferredLocations(any(LocationPreferenceConstraint.class))).thenCallRealMethod();

	return execution;
}
 
Example #14
Source File: OperatorGroupTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testGenerateScopeDefault() throws Exception {
	TaskManagerMetricGroup tmGroup = new TaskManagerMetricGroup(registry, "theHostName", "test-tm-id");
	TaskManagerJobMetricGroup jmGroup = new TaskManagerJobMetricGroup(registry, tmGroup, new JobID(), "myJobName");
	TaskMetricGroup taskGroup = new TaskMetricGroup(
			registry, jmGroup,  new JobVertexID(),  new AbstractID(), "aTaskName", 11, 0);
	OperatorMetricGroup opGroup = new OperatorMetricGroup(registry, taskGroup, new OperatorID(), "myOpName");

	assertArrayEquals(
			new String[] { "theHostName", "taskmanager", "test-tm-id", "myJobName", "myOpName", "11" },
			opGroup.getScopeComponents());

	assertEquals(
			"theHostName.taskmanager.test-tm-id.myJobName.myOpName.11.name",
			opGroup.getMetricIdentifier("name"));
}
 
Example #15
Source File: SchedulerTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public int getNumberOfAvailableSlotsForGroup(SlotSharingGroupId slotSharingGroupId, JobVertexID jobVertexId) {
	final SlotSharingManager multiTaskSlotManager = slotSharingManagersMap.get(slotSharingGroupId);

	if (multiTaskSlotManager != null) {
		int availableSlots = 0;

		for (SlotSharingManager.MultiTaskSlot multiTaskSlot : multiTaskSlotManager.getResolvedRootSlots()) {
			if (!multiTaskSlot.contains(jobVertexId)) {
				availableSlots++;
			}
		}

		return availableSlots;
	} else {
		throw new FlinkRuntimeException("No MultiTaskSlotmanager registered under " + slotSharingGroupId + '.');
	}
}
 
Example #16
Source File: ExecutionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Nonnull
private JobVertex createNoOpJobVertex() {
	final JobVertex jobVertex = new JobVertex("Test vertex", new JobVertexID());
	jobVertex.setInvokableClass(NoOpInvokable.class);

	return jobVertex;
}
 
Example #17
Source File: TestingRestfulGateway.java    From flink with Apache License 2.0 5 votes vote down vote up
public TestingRestfulGateway(
		String address,
		String hostname,
		Function<JobID, CompletableFuture<Acknowledge>> cancelJobFunction,
		Function<JobID, CompletableFuture<ArchivedExecutionGraph>> requestJobFunction,
		Function<JobID, CompletableFuture<JobResult>> requestJobResultFunction,
		Function<JobID, CompletableFuture<JobStatus>> requestJobStatusFunction,
		Supplier<CompletableFuture<MultipleJobsDetails>> requestMultipleJobDetailsSupplier,
		Supplier<CompletableFuture<ClusterOverview>> requestClusterOverviewSupplier,
		Supplier<CompletableFuture<Collection<String>>> requestMetricQueryServiceAddressesSupplier,
		Supplier<CompletableFuture<Collection<Tuple2<ResourceID, String>>>> requestTaskManagerMetricQueryServiceAddressesSupplier,
		BiFunction<JobID, JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOperatorBackPressureStatsFunction,
		BiFunction<JobID, String, CompletableFuture<String>> triggerSavepointFunction,
		BiFunction<JobID, String, CompletableFuture<String>> stopWithSavepointFunction) {
	this.address = address;
	this.hostname = hostname;
	this.cancelJobFunction = cancelJobFunction;
	this.requestJobFunction = requestJobFunction;
	this.requestJobResultFunction = requestJobResultFunction;
	this.requestJobStatusFunction = requestJobStatusFunction;
	this.requestMultipleJobDetailsSupplier = requestMultipleJobDetailsSupplier;
	this.requestClusterOverviewSupplier = requestClusterOverviewSupplier;
	this.requestMetricQueryServiceAddressesSupplier = requestMetricQueryServiceAddressesSupplier;
	this.requestTaskManagerMetricQueryServiceAddressesSupplier = requestTaskManagerMetricQueryServiceAddressesSupplier;
	this.requestOperatorBackPressureStatsFunction = requestOperatorBackPressureStatsFunction;
	this.triggerSavepointFunction = triggerSavepointFunction;
	this.stopWithSavepointFunction = stopWithSavepointFunction;
}
 
Example #18
Source File: PipelinedRegionExecutionViewTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test(expected = IllegalArgumentException.class)
public void finishingUnknownVertexThrowsException() {
	final PipelinedRegion from = PipelinedRegion.from(TEST_EXECUTION_VERTEX_ID);
	final PipelinedRegionExecutionView pipelinedRegionExecutionView = new PipelinedRegionExecutionView(from);

	final ExecutionVertexID unknownVertexId = new ExecutionVertexID(new JobVertexID(), 0);
	pipelinedRegionExecutionView.vertexFinished(unknownVertexId);
}
 
Example #19
Source File: DummyScheduledUnit.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public DummyScheduledUnit() {
	super(
		null,
		new JobVertexID(),
		null,
		null);
}
 
Example #20
Source File: TaskInformation.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public TaskInformation(
		JobVertexID jobVertexId,
		String taskName,
		int numberOfSubtasks,
		int maxNumberOfSubtaks,
		String invokableClassName,
		Configuration taskConfiguration) {
	this.jobVertexId = Preconditions.checkNotNull(jobVertexId);
	this.taskName = Preconditions.checkNotNull(taskName);
	this.numberOfSubtasks = Preconditions.checkNotNull(numberOfSubtasks);
	this.maxNumberOfSubtaks = Preconditions.checkNotNull(maxNumberOfSubtaks);
	this.invokableClassName = Preconditions.checkNotNull(invokableClassName);
	this.taskConfiguration = Preconditions.checkNotNull(taskConfiguration);
}
 
Example #21
Source File: KvStateLocationRegistryTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private ExecutionJobVertex createJobVertex(int maxParallelism) {
	JobVertexID id = new JobVertexID();
	ExecutionJobVertex vertex = mock(ExecutionJobVertex.class);

	when(vertex.getJobVertexId()).thenReturn(id);
	when(vertex.getMaxParallelism()).thenReturn(maxParallelism);

	return vertex;
}
 
Example #22
Source File: LegacyJobVertexIdTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntroduceLegacyJobVertexIds() throws Exception {
	JobVertexID defaultId = new JobVertexID();
	JobVertexID legacyId1 = new JobVertexID();
	JobVertexID legacyId2 = new JobVertexID();

	JobVertex jobVertex = new JobVertex("test", defaultId, Arrays.asList(legacyId1, legacyId2), new ArrayList<OperatorID>(), new ArrayList<OperatorID>());
	jobVertex.setInvokableClass(AbstractInvokable.class);

	ExecutionGraph executionGraph = new ExecutionGraph(
		mock(ScheduledExecutorService.class),
		mock(Executor.class),
		new JobID(),
		"test",
		mock(Configuration.class),
		mock(SerializedValue.class),
		Time.seconds(1),
		mock(RestartStrategy.class),
		mock(SlotProvider.class));

	ExecutionJobVertex executionJobVertex =
			new ExecutionJobVertex(executionGraph, jobVertex, 1, Time.seconds(1));

	Map<JobVertexID, ExecutionJobVertex> idToVertex = new HashMap<>();
	idToVertex.put(executionJobVertex.getJobVertexId(), executionJobVertex);

	Assert.assertEquals(executionJobVertex, idToVertex.get(defaultId));
	Assert.assertNull(idToVertex.get(legacyId1));
	Assert.assertNull(idToVertex.get(legacyId2));

	idToVertex = ExecutionJobVertex.includeLegacyJobVertexIDs(idToVertex);

	Assert.assertEquals(3, idToVertex.size());
	Assert.assertEquals(executionJobVertex, idToVertex.get(defaultId));
	Assert.assertEquals(executionJobVertex, idToVertex.get(legacyId1));
	Assert.assertEquals(executionJobVertex, idToVertex.get(legacyId2));
}
 
Example #23
Source File: CheckpointStatsTrackerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests access to the snapshotting settings.
 */
@Test
public void testGetSnapshottingSettings() throws Exception {
	ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
	when(jobVertex.getJobVertexId()).thenReturn(new JobVertexID());
	when(jobVertex.getParallelism()).thenReturn(1);

	JobCheckpointingSettings snapshottingSettings = new JobCheckpointingSettings(
		Collections.singletonList(new JobVertexID()),
		Collections.singletonList(new JobVertexID()),
		Collections.singletonList(new JobVertexID()),
		new CheckpointCoordinatorConfiguration(
			181238123L,
			19191992L,
			191929L,
			123,
			CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
			false
		),
		null);

	CheckpointStatsTracker tracker = new CheckpointStatsTracker(
		0,
		Collections.singletonList(jobVertex),
		snapshottingSettings.getCheckpointCoordinatorConfiguration(),
		new UnregisteredMetricsGroup());

	assertEquals(snapshottingSettings.getCheckpointCoordinatorConfiguration(), tracker.getJobCheckpointingConfiguration());
}
 
Example #24
Source File: StateAssignmentOperation.java    From flink with Apache License 2.0 5 votes vote down vote up
public StateAssignmentOperation(
	long restoreCheckpointId,
	Map<JobVertexID, ExecutionJobVertex> tasks,
	Map<OperatorID, OperatorState> operatorStates,
	boolean allowNonRestoredState) {

	this.restoreCheckpointId = restoreCheckpointId;
	this.tasks = Preconditions.checkNotNull(tasks);
	this.operatorStates = Preconditions.checkNotNull(operatorStates);
	this.allowNonRestoredState = allowNonRestoredState;
}
 
Example #25
Source File: StreamingJobGraphGeneratorNodeHashTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Verifies that each {@link JobVertexID} of the {@link JobGraph} is contained in the given map
 * and mapped to the same vertex name.
 */
private void verifyIdsEqual(JobGraph jobGraph, Map<JobVertexID, String> ids) {
	// Verify same number of vertices
	assertEquals(jobGraph.getNumberOfVertices(), ids.size());

	// Verify that all IDs->name mappings are identical
	for (JobVertex vertex : jobGraph.getVertices()) {
		String expectedName = ids.get(vertex.getID());
		assertNotNull(expectedName);
		assertEquals(expectedName, vertex.getName());
	}
}
 
Example #26
Source File: CheckpointStatsTracker.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new pending checkpoint tracker.
 *
 * @param checkpointId ID of the checkpoint.
 * @param triggerTimestamp Trigger timestamp of the checkpoint.
 * @param props The checkpoint properties.
 * @return Tracker for statistics gathering.
 */
PendingCheckpointStats reportPendingCheckpoint(
		long checkpointId,
		long triggerTimestamp,
		CheckpointProperties props) {

	ConcurrentHashMap<JobVertexID, TaskStateStats> taskStateStats = createEmptyTaskStateStatsMap();

	PendingCheckpointStats pending = new PendingCheckpointStats(
			checkpointId,
			triggerTimestamp,
			props,
			totalSubtaskCount,
			taskStateStats,
			new PendingCheckpointStatsCallback());

	statsReadWriteLock.lock();
	try {
		counts.incrementInProgressCheckpoints();
		history.addInProgressCheckpoint(pending);

		dirty = true;
	} finally {
		statsReadWriteLock.unlock();
	}

	return pending;
}
 
Example #27
Source File: ArchivedJobGenerationUtils.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private static void generateArchivedJob() throws Exception {
	// Attempt
	StringifiedAccumulatorResult acc1 = new StringifiedAccumulatorResult("name1", "type1", "value1");
	StringifiedAccumulatorResult acc2 = new StringifiedAccumulatorResult("name2", "type2", "value2");
	TaskManagerLocation location = new TaskManagerLocation(new ResourceID("hello"), InetAddress.getLocalHost(), 1234);
	AllocationID allocationID = new AllocationID(42L, 43L);
	originalAttempt = new ArchivedExecutionBuilder()
		.setStateTimestamps(new long[]{1, 2, 3, 4, 5, 6, 7, 8, 9})
		.setParallelSubtaskIndex(1)
		.setAttemptNumber(0)
		.setAssignedResourceLocation(location)
		.setAssignedAllocationID(allocationID)
		.setUserAccumulators(new StringifiedAccumulatorResult[]{acc1, acc2})
		.setState(ExecutionState.FINISHED)
		.setFailureCause("attemptException")
		.build();
	// Subtask
	originalSubtask = new ArchivedExecutionVertexBuilder()
		.setSubtaskIndex(originalAttempt.getParallelSubtaskIndex())
		.setTaskNameWithSubtask("hello(1/1)")
		.setCurrentExecution(originalAttempt)
		.build();
	// Task
	originalTask = new ArchivedExecutionJobVertexBuilder()
		.setTaskVertices(new ArchivedExecutionVertex[]{originalSubtask})
		.build();
	// Job
	Map<JobVertexID, ArchivedExecutionJobVertex> tasks = new HashMap<>();
	tasks.put(originalTask.getJobVertexId(), originalTask);
	originalJob = new ArchivedExecutionGraphBuilder()
		.setJobID(new JobID())
		.setTasks(tasks)
		.setFailureCause(new ErrorInfo(new Exception("jobException"), originalAttempt.getStateTimestamp(ExecutionState.FAILED)))
		.setState(JobStatus.FINISHED)
		.setStateTimestamps(new long[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})
		.setArchivedUserAccumulators(new StringifiedAccumulatorResult[]{acc1, acc2})
		.build();
}
 
Example #28
Source File: ScheduledUnit.java    From flink with Apache License 2.0 5 votes vote down vote up
public ScheduledUnit(
	@Nullable Execution task,
	JobVertexID jobVertexId,
	@Nullable SlotSharingGroupId slotSharingGroupId,
	@Nullable CoLocationConstraint coLocationConstraint) {

	this.vertexExecution = task;
	this.jobVertexId = Preconditions.checkNotNull(jobVertexId);
	this.slotSharingGroupId = slotSharingGroupId;
	this.coLocationConstraint = coLocationConstraint;

}
 
Example #29
Source File: DefaultExecutionSlotAllocatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that cancels an execution vertex which is not existed.
 */
@Test
public void testCancelNonExistingExecutionVertex() {
	final DefaultExecutionSlotAllocator executionSlotAllocator = createExecutionSlotAllocator();

	ExecutionVertexID inValidExecutionVertexId = new ExecutionVertexID(new JobVertexID(), 0);
	executionSlotAllocator.cancel(inValidExecutionVertexId);

	assertThat(slotProvider.getCancelledSlotRequestIds(), is(empty()));
}
 
Example #30
Source File: IntermediateResultPartitionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private static IntermediateResult createResult(
		ResultPartitionType resultPartitionType,
		int producerCount) throws Exception {

	ExecutionJobVertex jobVertex = getExecutionJobVertex(new JobVertexID(), new DirectScheduledExecutorService());
	IntermediateResult result =
			new IntermediateResult(new IntermediateDataSetID(), jobVertex, producerCount, resultPartitionType);
	for (int i = 0; i < producerCount; i++) {
		// Generate result partition in the result
		new ExecutionVertex(jobVertex, i, new IntermediateResult[]{result}, Time.minutes(1));
	}

	return result;
}