org.apache.flink.runtime.executiongraph.AccessExecution Java Examples

The following examples show how to use org.apache.flink.runtime.executiongraph.AccessExecution. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AbstractSubtaskAttemptHandler.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
protected R handleRequest(HandlerRequest<EmptyRequestBody, M> request, AccessExecutionVertex executionVertex) throws RestHandlerException {
	final Integer attemptNumber = request.getPathParameter(SubtaskAttemptPathParameter.class);

	final AccessExecution currentAttempt = executionVertex.getCurrentExecutionAttempt();
	if (attemptNumber == currentAttempt.getAttemptNumber()) {
		return handleRequest(request, currentAttempt);
	} else if (attemptNumber >= 0 && attemptNumber < currentAttempt.getAttemptNumber()) {
		final AccessExecution execution = executionVertex.getPriorExecutionAttempt(attemptNumber);

		if (execution != null) {
			return handleRequest(request, execution);
		} else {
			throw new RestHandlerException("Attempt " + attemptNumber + " not found in subtask " +
				executionVertex.getTaskNameWithSubtaskIndex(), HttpResponseStatus.NOT_FOUND);
		}
	} else {
		throw new RestHandlerException("Invalid attempt num " + attemptNumber, HttpResponseStatus.NOT_FOUND);
	}
}
Example #2
Source File: SubtaskCurrentAttemptDetailsHandler.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
protected SubtaskExecutionAttemptDetailsInfo handleRequest(
		HandlerRequest<EmptyRequestBody, SubtaskMessageParameters> request,
		AccessExecutionVertex executionVertex) throws RestHandlerException {

	final AccessExecution execution = executionVertex.getCurrentExecutionAttempt();

	final MutableIOMetrics ioMetrics = new MutableIOMetrics();

	final JobID jobID = request.getPathParameter(JobIDPathParameter.class);
	final JobVertexID jobVertexID = request.getPathParameter(JobVertexIdPathParameter.class);

	ioMetrics.addIOMetrics(
		execution,
		metricFetcher,
		jobID.toString(),
		jobVertexID.toString()
	);

	return SubtaskExecutionAttemptDetailsInfo.create(execution, ioMetrics);
}
Example #3
Source File: SubtaskExecutionAttemptDetailsHandler.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static SubtaskExecutionAttemptDetailsInfo createDetailsInfo(
		AccessExecution execution,
		JobID jobID,
		JobVertexID jobVertexID,
		@Nullable MetricFetcher metricFetcher) {
	final MutableIOMetrics ioMetrics = new MutableIOMetrics();

	ioMetrics.addIOMetrics(
		execution,
		metricFetcher,
		jobID.toString(),
		jobVertexID.toString()
	);

	return SubtaskExecutionAttemptDetailsInfo.create(execution, ioMetrics);
}
Example #4
Source File: SubtaskExecutionAttemptDetailsHandler.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private static SubtaskExecutionAttemptDetailsInfo createDetailsInfo(
		AccessExecution execution,
		JobID jobID,
		JobVertexID jobVertexID,
		@Nullable MetricFetcher metricFetcher) {
	final MutableIOMetrics ioMetrics = new MutableIOMetrics();

	ioMetrics.addIOMetrics(
		execution,
		metricFetcher,
		jobID.toString(),
		jobVertexID.toString()
	);

	return SubtaskExecutionAttemptDetailsInfo.create(execution, ioMetrics);
}
Example #5
Source File: JobVertexDetailsHandler.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static JobVertexDetailsInfo createJobVertexDetailsInfo(AccessExecutionJobVertex jobVertex, JobID jobID, @Nullable MetricFetcher metricFetcher) {
	List<SubtaskExecutionAttemptDetailsInfo> subtasks = new ArrayList<>();
	final long now = System.currentTimeMillis();
	for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) {
		final AccessExecution execution = vertex.getCurrentExecutionAttempt();
		final JobVertexID jobVertexID = jobVertex.getJobVertexId();
		subtasks.add(SubtaskExecutionAttemptDetailsInfo.create(execution, metricFetcher, jobID, jobVertexID));
	}

	return new JobVertexDetailsInfo(
		jobVertex.getJobVertexId(),
		jobVertex.getName(),
		jobVertex.getParallelism(),
		now,
		subtasks);
}
Example #6
Source File: SubtaskCurrentAttemptDetailsHandler.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
protected SubtaskExecutionAttemptDetailsInfo handleRequest(
		HandlerRequest<EmptyRequestBody, SubtaskMessageParameters> request,
		AccessExecutionVertex executionVertex) throws RestHandlerException {

	final AccessExecution execution = executionVertex.getCurrentExecutionAttempt();

	final MutableIOMetrics ioMetrics = new MutableIOMetrics();

	final JobID jobID = request.getPathParameter(JobIDPathParameter.class);
	final JobVertexID jobVertexID = request.getPathParameter(JobVertexIdPathParameter.class);

	ioMetrics.addIOMetrics(
		execution,
		metricFetcher,
		jobID.toString(),
		jobVertexID.toString()
	);

	return SubtaskExecutionAttemptDetailsInfo.create(execution, ioMetrics);
}
Example #7
Source File: AbstractSubtaskAttemptHandler.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
protected R handleRequest(HandlerRequest<EmptyRequestBody, M> request, AccessExecutionVertex executionVertex) throws RestHandlerException {
	final Integer attemptNumber = request.getPathParameter(SubtaskAttemptPathParameter.class);

	final AccessExecution currentAttempt = executionVertex.getCurrentExecutionAttempt();
	if (attemptNumber == currentAttempt.getAttemptNumber()) {
		return handleRequest(request, currentAttempt);
	} else if (attemptNumber >= 0 && attemptNumber < currentAttempt.getAttemptNumber()) {
		final AccessExecution execution = executionVertex.getPriorExecutionAttempt(attemptNumber);

		if (execution != null) {
			return handleRequest(request, execution);
		} else {
			throw new RestHandlerException("Attempt " + attemptNumber + " not found in subtask " +
				executionVertex.getTaskNameWithSubtaskIndex(), HttpResponseStatus.NOT_FOUND);
		}
	} else {
		throw new RestHandlerException("Invalid attempt num " + attemptNumber, HttpResponseStatus.NOT_FOUND);
	}
}
Example #8
Source File: AbstractSubtaskAttemptHandler.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
protected R handleRequest(HandlerRequest<EmptyRequestBody, M> request, AccessExecutionVertex executionVertex) throws RestHandlerException {
	final Integer attemptNumber = request.getPathParameter(SubtaskAttemptPathParameter.class);

	final AccessExecution currentAttempt = executionVertex.getCurrentExecutionAttempt();
	if (attemptNumber == currentAttempt.getAttemptNumber()) {
		return handleRequest(request, currentAttempt);
	} else if (attemptNumber >= 0 && attemptNumber < currentAttempt.getAttemptNumber()) {
		final AccessExecution execution = executionVertex.getPriorExecutionAttempt(attemptNumber);

		if (execution != null) {
			return handleRequest(request, execution);
		} else {
			throw new RestHandlerException("Attempt " + attemptNumber + " not found in subtask " +
				executionVertex.getTaskNameWithSubtaskIndex(), HttpResponseStatus.NOT_FOUND);
		}
	} else {
		throw new RestHandlerException("Invalid attempt num " + attemptNumber, HttpResponseStatus.NOT_FOUND);
	}
}
Example #9
Source File: SubtaskExecutionAttemptAccumulatorsHandler.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static SubtaskExecutionAttemptAccumulatorsInfo createAccumulatorInfo(AccessExecution execution) {
	final StringifiedAccumulatorResult[] accs = execution.getUserAccumulatorsStringified();
	final ArrayList<UserAccumulator> userAccumulatorList = new ArrayList<>(accs.length);

	for (StringifiedAccumulatorResult acc : accs) {
		userAccumulatorList.add(new UserAccumulator(acc.getName(), acc.getType(), acc.getValue()));
	}

	return new SubtaskExecutionAttemptAccumulatorsInfo(
		execution.getParallelSubtaskIndex(),
		execution.getAttemptNumber(),
		execution.getAttemptId().toString(),
		userAccumulatorList);
}
Example #10
Source File: TaskExecutorITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
private SupplierWithException<Boolean, Exception> jobIsRunning(Supplier<CompletableFuture<? extends AccessExecutionGraph>> executionGraphFutureSupplier) {
	final Predicate<AccessExecution> runningOrFinished = ExecutionGraphTestUtils.isInExecutionState(ExecutionState.RUNNING).or(ExecutionGraphTestUtils.isInExecutionState(ExecutionState.FINISHED));
	final Predicate<AccessExecutionGraph> allExecutionsRunning = ExecutionGraphTestUtils.allExecutionsPredicate(runningOrFinished);

	return () -> {
		final AccessExecutionGraph executionGraph = executionGraphFutureSupplier.get().join();
		return allExecutionsRunning.test(executionGraph);
	};
}
Example #11
Source File: JobMasterTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static Collection<AccessExecution> getExecutions(final JobMasterGateway jobMasterGateway) {
	final ArchivedExecutionGraph archivedExecutionGraph = requestExecutionGraph(jobMasterGateway);

	return archivedExecutionGraph.getAllVertices().values()
		.stream()
		.flatMap(vertex -> Arrays.stream(vertex.getTaskVertices()))
		.map(AccessExecutionVertex::getCurrentExecutionAttempt)
		.collect(Collectors.toList());
}
Example #12
Source File: JobMasterTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static List<AccessExecution> getExecutions(final JobMasterGateway jobMasterGateway, final JobVertexID jobVertexId) {
	final ArchivedExecutionGraph archivedExecutionGraph = requestExecutionGraph(jobMasterGateway);

	return Optional.ofNullable(archivedExecutionGraph.getAllVertices().get(jobVertexId))
		.map(accessExecutionJobVertex -> Arrays.asList(accessExecutionJobVertex.getTaskVertices()))
		.orElse(Collections.emptyList())
		.stream()
		.map(AccessExecutionVertex::getCurrentExecutionAttempt)
		.collect(Collectors.toList());
}
Example #13
Source File: TaskExecutorITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
private SupplierWithException<Boolean, Exception> jobIsRunning(Supplier<CompletableFuture<? extends AccessExecutionGraph>> executionGraphFutureSupplier) {
	final Predicate<AccessExecution> runningOrFinished = ExecutionGraphTestUtils.isInExecutionState(ExecutionState.RUNNING).or(ExecutionGraphTestUtils.isInExecutionState(ExecutionState.FINISHED));
	final Predicate<AccessExecutionGraph> allExecutionsRunning = ExecutionGraphTestUtils.allExecutionsPredicate(runningOrFinished);

	return () -> {
		final AccessExecutionGraph executionGraph = executionGraphFutureSupplier.get().join();
		return allExecutionsRunning.test(executionGraph);
	};
}
Example #14
Source File: SubtaskCurrentAttemptDetailsHandler.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected SubtaskExecutionAttemptDetailsInfo handleRequest(
		HandlerRequest<EmptyRequestBody, SubtaskMessageParameters> request,
		AccessExecutionVertex executionVertex) throws RestHandlerException {

	final AccessExecution execution = executionVertex.getCurrentExecutionAttempt();

	final JobID jobID = request.getPathParameter(JobIDPathParameter.class);
	final JobVertexID jobVertexID = request.getPathParameter(JobVertexIdPathParameter.class);

	return SubtaskExecutionAttemptDetailsInfo.create(execution, metricFetcher, jobID, jobVertexID);
}
Example #15
Source File: SubtaskExecutionAttemptDetailsHandler.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected SubtaskExecutionAttemptDetailsInfo handleRequest(
		HandlerRequest<EmptyRequestBody, SubtaskAttemptMessageParameters> request,
		AccessExecution execution) throws RestHandlerException {

	final JobID jobID = request.getPathParameter(JobIDPathParameter.class);
	final JobVertexID jobVertexID = request.getPathParameter(JobVertexIdPathParameter.class);

	return SubtaskExecutionAttemptDetailsInfo.create(execution, metricFetcher, jobID, jobVertexID);
}
Example #16
Source File: SubtaskExecutionAttemptDetailsHandler.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException {
	List<ArchivedJson> archive = new ArrayList<>(16);
	for (AccessExecutionJobVertex task : graph.getAllVertices().values()) {
		for (AccessExecutionVertex subtask : task.getTaskVertices()) {
			ResponseBody curAttemptJson = SubtaskExecutionAttemptDetailsInfo.create(subtask.getCurrentExecutionAttempt(), null, graph.getJobID(), task.getJobVertexId());
			String curAttemptPath = getMessageHeaders().getTargetRestEndpointURL()
				.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString())
				.replace(':' + JobVertexIdPathParameter.KEY, task.getJobVertexId().toString())
				.replace(':' + SubtaskIndexPathParameter.KEY, String.valueOf(subtask.getParallelSubtaskIndex()))
				.replace(':' + SubtaskAttemptPathParameter.KEY, String.valueOf(subtask.getCurrentExecutionAttempt().getAttemptNumber()));

			archive.add(new ArchivedJson(curAttemptPath, curAttemptJson));

			for (int x = 0; x < subtask.getCurrentExecutionAttempt().getAttemptNumber(); x++) {
				AccessExecution attempt = subtask.getPriorExecutionAttempt(x);
				if (attempt != null) {
					ResponseBody json = SubtaskExecutionAttemptDetailsInfo.create(attempt, null, graph.getJobID(), task.getJobVertexId());
					String path = getMessageHeaders().getTargetRestEndpointURL()
						.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString())
						.replace(':' + JobVertexIdPathParameter.KEY, task.getJobVertexId().toString())
						.replace(':' + SubtaskIndexPathParameter.KEY, String.valueOf(subtask.getParallelSubtaskIndex()))
						.replace(':' + SubtaskAttemptPathParameter.KEY, String.valueOf(attempt.getAttemptNumber()));
					archive.add(new ArchivedJson(path, json));
				}
			}
		}
	}
	return archive;
}
Example #17
Source File: SubtaskExecutionAttemptAccumulatorsHandler.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException {
	List<ArchivedJson> archive = new ArrayList<>(16);
	for (AccessExecutionJobVertex task : graph.getAllVertices().values()) {
		for (AccessExecutionVertex subtask : task.getTaskVertices()) {
			ResponseBody curAttemptJson = createAccumulatorInfo(subtask.getCurrentExecutionAttempt());
			String curAttemptPath = getMessageHeaders().getTargetRestEndpointURL()
				.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString())
				.replace(':' + JobVertexIdPathParameter.KEY, task.getJobVertexId().toString())
				.replace(':' + SubtaskIndexPathParameter.KEY, String.valueOf(subtask.getParallelSubtaskIndex()))
				.replace(':' + SubtaskAttemptPathParameter.KEY, String.valueOf(subtask.getCurrentExecutionAttempt().getAttemptNumber()));

			archive.add(new ArchivedJson(curAttemptPath, curAttemptJson));

			for (int x = 0; x < subtask.getCurrentExecutionAttempt().getAttemptNumber(); x++) {
				AccessExecution attempt = subtask.getPriorExecutionAttempt(x);
				if (attempt != null){
					ResponseBody json = createAccumulatorInfo(attempt);
					String path = getMessageHeaders().getTargetRestEndpointURL()
						.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString())
						.replace(':' + JobVertexIdPathParameter.KEY, task.getJobVertexId().toString())
						.replace(':' + SubtaskIndexPathParameter.KEY, String.valueOf(subtask.getParallelSubtaskIndex()))
						.replace(':' + SubtaskAttemptPathParameter.KEY, String.valueOf(attempt.getAttemptNumber()));
					archive.add(new ArchivedJson(path, json));
				}
			}
		}
	}
	return archive;
}
Example #18
Source File: SubtaskExecutionAttemptAccumulatorsHandler.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static SubtaskExecutionAttemptAccumulatorsInfo createAccumulatorInfo(AccessExecution execution) {
	final StringifiedAccumulatorResult[] accs = execution.getUserAccumulatorsStringified();
	final ArrayList<UserAccumulator> userAccumulatorList = new ArrayList<>(accs.length);

	for (StringifiedAccumulatorResult acc : accs) {
		userAccumulatorList.add(new UserAccumulator(acc.getName(), acc.getType(), acc.getValue()));
	}

	return new SubtaskExecutionAttemptAccumulatorsInfo(
		execution.getParallelSubtaskIndex(),
		execution.getAttemptNumber(),
		execution.getAttemptId().toString(),
		userAccumulatorList);
}
Example #19
Source File: JobMasterTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static Collection<AccessExecution> getExecutions(final JobMasterGateway jobMasterGateway) {
	final ArchivedExecutionGraph archivedExecutionGraph = requestExecutionGraph(jobMasterGateway);

	return archivedExecutionGraph.getAllVertices().values()
		.stream()
		.flatMap(vertex -> Arrays.stream(vertex.getTaskVertices()))
		.map(AccessExecutionVertex::getCurrentExecutionAttempt)
		.collect(Collectors.toList());
}
Example #20
Source File: JobMasterTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static List<AccessExecution> getExecutions(final JobMasterGateway jobMasterGateway, final JobVertexID jobVertexId) {
	final ArchivedExecutionGraph archivedExecutionGraph = requestExecutionGraph(jobMasterGateway);

	return Optional.ofNullable(archivedExecutionGraph.getAllVertices().get(jobVertexId))
		.map(accessExecutionJobVertex -> Arrays.asList(accessExecutionJobVertex.getTaskVertices()))
		.orElse(Collections.emptyList())
		.stream()
		.map(AccessExecutionVertex::getCurrentExecutionAttempt)
		.collect(Collectors.toList());
}
Example #21
Source File: SubtaskExecutionAttemptDetailsInfo.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public static SubtaskExecutionAttemptDetailsInfo create(AccessExecution execution, MutableIOMetrics ioMetrics) {
	final ExecutionState status = execution.getState();
	final long now = System.currentTimeMillis();

	final TaskManagerLocation location = execution.getAssignedResourceLocation();
	final String locationString = location == null ? "(unassigned)" : location.getHostname();

	long startTime = execution.getStateTimestamp(ExecutionState.DEPLOYING);
	if (startTime == 0) {
		startTime = -1;
	}
	final long endTime = status.isTerminal() ? execution.getStateTimestamp(status) : -1;
	final long duration = startTime > 0 ? ((endTime > 0 ? endTime : now) - startTime) : -1;

	final IOMetricsInfo ioMetricsInfo = new IOMetricsInfo(
		ioMetrics.getNumBytesInLocal() + ioMetrics.getNumBytesInRemote(),
		ioMetrics.isNumBytesInLocalComplete() && ioMetrics.isNumBytesInRemoteComplete(),
		ioMetrics.getNumBytesOut(),
		ioMetrics.isNumBytesOutComplete(),
		ioMetrics.getNumRecordsIn(),
		ioMetrics.isNumRecordsInComplete(),
		ioMetrics.getNumRecordsOut(),
		ioMetrics.isNumRecordsOutComplete());

	return new SubtaskExecutionAttemptDetailsInfo(
		execution.getParallelSubtaskIndex(),
		status,
		execution.getAttemptNumber(),
		locationString,
		startTime,
		endTime,
		duration,
		ioMetricsInfo
	);
}
Example #22
Source File: SubtaskExecutionAttemptAccumulatorsHandler.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException {
	List<ArchivedJson> archive = new ArrayList<>(16);
	for (AccessExecutionJobVertex task : graph.getAllVertices().values()) {
		for (AccessExecutionVertex subtask : task.getTaskVertices()) {
			ResponseBody curAttemptJson = createAccumulatorInfo(subtask.getCurrentExecutionAttempt());
			String curAttemptPath = getMessageHeaders().getTargetRestEndpointURL()
				.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString())
				.replace(':' + JobVertexIdPathParameter.KEY, task.getJobVertexId().toString())
				.replace(':' + SubtaskIndexPathParameter.KEY, String.valueOf(subtask.getParallelSubtaskIndex()))
				.replace(':' + SubtaskAttemptPathParameter.KEY, String.valueOf(subtask.getCurrentExecutionAttempt().getAttemptNumber()));

			archive.add(new ArchivedJson(curAttemptPath, curAttemptJson));

			for (int x = 0; x < subtask.getCurrentExecutionAttempt().getAttemptNumber(); x++) {
				AccessExecution attempt = subtask.getPriorExecutionAttempt(x);
				if (attempt != null){
					ResponseBody json = createAccumulatorInfo(attempt);
					String path = getMessageHeaders().getTargetRestEndpointURL()
						.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString())
						.replace(':' + JobVertexIdPathParameter.KEY, task.getJobVertexId().toString())
						.replace(':' + SubtaskIndexPathParameter.KEY, String.valueOf(subtask.getParallelSubtaskIndex()))
						.replace(':' + SubtaskAttemptPathParameter.KEY, String.valueOf(attempt.getAttemptNumber()));
					archive.add(new ArchivedJson(path, json));
				}
			}
		}
	}
	return archive;
}
Example #23
Source File: SubtaskExecutionAttemptDetailsHandler.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
protected SubtaskExecutionAttemptDetailsInfo handleRequest(
		HandlerRequest<EmptyRequestBody, SubtaskAttemptMessageParameters> request,
		AccessExecution execution) throws RestHandlerException {

	final JobID jobID = request.getPathParameter(JobIDPathParameter.class);
	final JobVertexID jobVertexID = request.getPathParameter(JobVertexIdPathParameter.class);

	return createDetailsInfo(execution, jobID, jobVertexID, metricFetcher);
}
Example #24
Source File: SubtaskExecutionAttemptDetailsHandler.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException {
	List<ArchivedJson> archive = new ArrayList<>(16);
	for (AccessExecutionJobVertex task : graph.getAllVertices().values()) {
		for (AccessExecutionVertex subtask : task.getTaskVertices()) {
			ResponseBody curAttemptJson = createDetailsInfo(subtask.getCurrentExecutionAttempt(), graph.getJobID(), task.getJobVertexId(), null);
			String curAttemptPath = getMessageHeaders().getTargetRestEndpointURL()
				.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString())
				.replace(':' + JobVertexIdPathParameter.KEY, task.getJobVertexId().toString())
				.replace(':' + SubtaskIndexPathParameter.KEY, String.valueOf(subtask.getParallelSubtaskIndex()))
				.replace(':' + SubtaskAttemptPathParameter.KEY, String.valueOf(subtask.getCurrentExecutionAttempt().getAttemptNumber()));

			archive.add(new ArchivedJson(curAttemptPath, curAttemptJson));

			for (int x = 0; x < subtask.getCurrentExecutionAttempt().getAttemptNumber(); x++) {
				AccessExecution attempt = subtask.getPriorExecutionAttempt(x);
				if (attempt != null) {
					ResponseBody json = createDetailsInfo(attempt, graph.getJobID(), task.getJobVertexId(), null);
					String path = getMessageHeaders().getTargetRestEndpointURL()
						.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString())
						.replace(':' + JobVertexIdPathParameter.KEY, task.getJobVertexId().toString())
						.replace(':' + SubtaskIndexPathParameter.KEY, String.valueOf(subtask.getParallelSubtaskIndex()))
						.replace(':' + SubtaskAttemptPathParameter.KEY, String.valueOf(attempt.getAttemptNumber()));
					archive.add(new ArchivedJson(path, json));
				}
			}
		}
	}
	return archive;
}
Example #25
Source File: SubtaskExecutionAttemptDetailsHandler.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException {
	List<ArchivedJson> archive = new ArrayList<>(16);
	for (AccessExecutionJobVertex task : graph.getAllVertices().values()) {
		for (AccessExecutionVertex subtask : task.getTaskVertices()) {
			ResponseBody curAttemptJson = createDetailsInfo(subtask.getCurrentExecutionAttempt(), graph.getJobID(), task.getJobVertexId(), null);
			String curAttemptPath = getMessageHeaders().getTargetRestEndpointURL()
				.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString())
				.replace(':' + JobVertexIdPathParameter.KEY, task.getJobVertexId().toString())
				.replace(':' + SubtaskIndexPathParameter.KEY, String.valueOf(subtask.getParallelSubtaskIndex()))
				.replace(':' + SubtaskAttemptPathParameter.KEY, String.valueOf(subtask.getCurrentExecutionAttempt().getAttemptNumber()));

			archive.add(new ArchivedJson(curAttemptPath, curAttemptJson));

			for (int x = 0; x < subtask.getCurrentExecutionAttempt().getAttemptNumber(); x++) {
				AccessExecution attempt = subtask.getPriorExecutionAttempt(x);
				if (attempt != null) {
					ResponseBody json = createDetailsInfo(attempt, graph.getJobID(), task.getJobVertexId(), null);
					String path = getMessageHeaders().getTargetRestEndpointURL()
						.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString())
						.replace(':' + JobVertexIdPathParameter.KEY, task.getJobVertexId().toString())
						.replace(':' + SubtaskIndexPathParameter.KEY, String.valueOf(subtask.getParallelSubtaskIndex()))
						.replace(':' + SubtaskAttemptPathParameter.KEY, String.valueOf(attempt.getAttemptNumber()));
					archive.add(new ArchivedJson(path, json));
				}
			}
		}
	}
	return archive;
}
Example #26
Source File: SubtaskExecutionAttemptDetailsHandler.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected SubtaskExecutionAttemptDetailsInfo handleRequest(
		HandlerRequest<EmptyRequestBody, SubtaskAttemptMessageParameters> request,
		AccessExecution execution) throws RestHandlerException {

	final JobID jobID = request.getPathParameter(JobIDPathParameter.class);
	final JobVertexID jobVertexID = request.getPathParameter(JobVertexIdPathParameter.class);

	return createDetailsInfo(execution, jobID, jobVertexID, metricFetcher);
}
Example #27
Source File: SubtaskExecutionAttemptDetailsInfo.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static SubtaskExecutionAttemptDetailsInfo create(AccessExecution execution, MutableIOMetrics ioMetrics) {
	final ExecutionState status = execution.getState();
	final long now = System.currentTimeMillis();

	final TaskManagerLocation location = execution.getAssignedResourceLocation();
	final String locationString = location == null ? "(unassigned)" : location.getHostname();

	long startTime = execution.getStateTimestamp(ExecutionState.DEPLOYING);
	if (startTime == 0) {
		startTime = -1;
	}
	final long endTime = status.isTerminal() ? execution.getStateTimestamp(status) : -1;
	final long duration = startTime > 0 ? ((endTime > 0 ? endTime : now) - startTime) : -1;

	final IOMetricsInfo ioMetricsInfo = new IOMetricsInfo(
		ioMetrics.getNumBytesIn(),
		ioMetrics.isNumBytesInComplete(),
		ioMetrics.getNumBytesOut(),
		ioMetrics.isNumBytesOutComplete(),
		ioMetrics.getNumRecordsIn(),
		ioMetrics.isNumRecordsInComplete(),
		ioMetrics.getNumRecordsOut(),
		ioMetrics.isNumRecordsOutComplete());

	return new SubtaskExecutionAttemptDetailsInfo(
		execution.getParallelSubtaskIndex(),
		status,
		execution.getAttemptNumber(),
		locationString,
		startTime,
		endTime,
		duration,
		ioMetricsInfo
	);
}
Example #28
Source File: ArchivedJobGenerationUtils.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public static AccessExecution getTestAttempt() throws Exception {
	synchronized (lock) {
		if (originalJob == null) {
			generateArchivedJob();
		}
	}
	return originalAttempt;
}
Example #29
Source File: TaskExecutorITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private SupplierWithException<Boolean, Exception> jobIsRunning(Supplier<CompletableFuture<? extends AccessExecutionGraph>> executionGraphFutureSupplier) {
	final Predicate<AccessExecution> runningOrFinished = ExecutionGraphTestUtils.isInExecutionState(ExecutionState.RUNNING).or(ExecutionGraphTestUtils.isInExecutionState(ExecutionState.FINISHED));
	final Predicate<AccessExecutionGraph> allExecutionsRunning = ExecutionGraphTestUtils.allExecutionsPredicate(runningOrFinished);

	return () -> {
		final AccessExecutionGraph executionGraph = executionGraphFutureSupplier.get().join();
		return allExecutionsRunning.test(executionGraph);
	};
}
Example #30
Source File: SubtaskExecutionAttemptAccumulatorsHandler.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException {
	List<ArchivedJson> archive = new ArrayList<>(16);
	for (AccessExecutionJobVertex task : graph.getAllVertices().values()) {
		for (AccessExecutionVertex subtask : task.getTaskVertices()) {
			ResponseBody curAttemptJson = createAccumulatorInfo(subtask.getCurrentExecutionAttempt());
			String curAttemptPath = getMessageHeaders().getTargetRestEndpointURL()
				.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString())
				.replace(':' + JobVertexIdPathParameter.KEY, task.getJobVertexId().toString())
				.replace(':' + SubtaskIndexPathParameter.KEY, String.valueOf(subtask.getParallelSubtaskIndex()))
				.replace(':' + SubtaskAttemptPathParameter.KEY, String.valueOf(subtask.getCurrentExecutionAttempt().getAttemptNumber()));

			archive.add(new ArchivedJson(curAttemptPath, curAttemptJson));

			for (int x = 0; x < subtask.getCurrentExecutionAttempt().getAttemptNumber(); x++) {
				AccessExecution attempt = subtask.getPriorExecutionAttempt(x);
				if (attempt != null){
					ResponseBody json = createAccumulatorInfo(attempt);
					String path = getMessageHeaders().getTargetRestEndpointURL()
						.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString())
						.replace(':' + JobVertexIdPathParameter.KEY, task.getJobVertexId().toString())
						.replace(':' + SubtaskIndexPathParameter.KEY, String.valueOf(subtask.getParallelSubtaskIndex()))
						.replace(':' + SubtaskAttemptPathParameter.KEY, String.valueOf(attempt.getAttemptNumber()));
					archive.add(new ArchivedJson(path, json));
				}
			}
		}
	}
	return archive;
}