Java Code Examples for org.apache.flink.runtime.execution.ExecutionState#FAILED

private void notifyCoordinatorOfCancellation(ExecutionVertex vertex) {
	// this method makes a best effort to filter out duplicate notifications, meaning cases where
	// the coordinator was already notified for that specific task
	// we don't notify if the task is already FAILED, CANCELLING, or CANCELED

	final ExecutionState currentState = vertex.getExecutionState();
	if (currentState == ExecutionState.FAILED ||
			currentState == ExecutionState.CANCELING ||
			currentState == ExecutionState.CANCELED) {
		return;
	}

	for (OperatorCoordinator coordinator : vertex.getJobVertex().getOperatorCoordinators()) {
		coordinator.subtaskFailed(vertex.getParallelSubtaskIndex(), null);
	}
}
 

@Test
public void updateTaskExecutionStateReturnsFalseIfExecutionDoesNotExist() {
	final JobGraph jobGraph = singleNonParallelJobVertexJobGraph();
	final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);

	final TaskExecutionState taskExecutionState = new TaskExecutionState(
		jobGraph.getJobID(),
		new ExecutionAttemptID(),
		ExecutionState.FAILED);

	assertFalse(scheduler.updateTaskExecutionState(taskExecutionState));
}
 

/**
 * A utility function that computes an "aggregated" state for the vertex.
 *
 * <p>This state is not used anywhere in the  coordination, but can be used for display
 * in dashboards to as a summary for how the particular parallel operation represented by
 * this ExecutionJobVertex is currently behaving.
 *
 * <p>For example, if at least one parallel task is failed, the aggregate state is failed.
 * If not, and at least one parallel task is cancelling (or cancelled), the aggregate state
 * is cancelling (or cancelled). If all tasks are finished, the aggregate state is finished,
 * and so on.
 *
 * @param verticesPerState The number of vertices in each state (indexed by the ordinal of
 *                         the ExecutionState values).
 * @param parallelism The parallelism of the ExecutionJobVertex
 *
 * @return The aggregate state of this ExecutionJobVertex.
 */
public static ExecutionState getAggregateJobVertexState(int[] verticesPerState, int parallelism) {
	if (verticesPerState == null || verticesPerState.length != ExecutionState.values().length) {
		throw new IllegalArgumentException("Must provide an array as large as there are execution states.");
	}

	if (verticesPerState[ExecutionState.FAILED.ordinal()] > 0) {
		return ExecutionState.FAILED;
	}
	if (verticesPerState[ExecutionState.CANCELING.ordinal()] > 0) {
		return ExecutionState.CANCELING;
	}
	else if (verticesPerState[ExecutionState.CANCELED.ordinal()] > 0) {
		return ExecutionState.CANCELED;
	}
	else if (verticesPerState[ExecutionState.RUNNING.ordinal()] > 0) {
		return ExecutionState.RUNNING;
	}
	else if (verticesPerState[ExecutionState.FINISHED.ordinal()] > 0) {
		return verticesPerState[ExecutionState.FINISHED.ordinal()] == parallelism ?
				ExecutionState.FINISHED : ExecutionState.RUNNING;
	}
	else {
		// all else collapses under created
		return ExecutionState.CREATED;
	}
}
 

private void abortConsumptionOrIgnoreCheckResult(ResponseHandle responseHandle) {
	ExecutionState producerState = getProducerState(responseHandle);
	if (producerState == ExecutionState.CANCELING ||
		producerState == ExecutionState.CANCELED ||
		producerState == ExecutionState.FAILED) {

		// The producing execution has been canceled or failed. We
		// don't need to re-trigger the request since it cannot
		// succeed.
		if (LOG.isDebugEnabled()) {
			LOG.debug("Cancelling task {} after the producer of partition {} with attempt ID {} has entered state {}.",
				taskNameWithSubtask,
				resultPartitionId.getPartitionId(),
				resultPartitionId.getProducerId(),
				producerState);
		}

		responseHandle.cancelConsumption();
	} else {
		// Any other execution state is unexpected. Currently, only
		// state CREATED is left out of the checked states. If we
		// see a producer in this state, something went wrong with
		// scheduling in topological order.
		final String msg = String.format("Producer with attempt ID %s of partition %s in unexpected state %s.",
			resultPartitionId.getProducerId(),
			resultPartitionId.getPartitionId(),
			producerState);

		responseHandle.failConsumption(new IllegalStateException(msg));
	}
}
 

@Test
public void testDeployFailedAsynchronously() {
	try {
		final JobVertexID jid = new JobVertexID();
		final ExecutionJobVertex ejv = getExecutionVertex(jid);
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
			AkkaUtils.getDefaultTimeout());

		final LogicalSlot slot = new TestingLogicalSlot(new SubmitFailingSimpleAckingTaskManagerGateway());

		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());

		vertex.deployToSlot(slot);

		// wait until the state transition must be done
		for (int i = 0; i < 100; i++) {
			if (vertex.getExecutionState() == ExecutionState.FAILED && vertex.getFailureCause() != null) {
				break;
			} else {
				Thread.sleep(10);
			}
		}

		assertEquals(ExecutionState.FAILED, vertex.getExecutionState());
		assertNotNull(vertex.getFailureCause());
		assertTrue(vertex.getFailureCause().getMessage().contains(ERROR_MESSAGE));

		assertTrue(vertex.getStateTimestamp(ExecutionState.CREATED) > 0);
		assertTrue(vertex.getStateTimestamp(ExecutionState.DEPLOYING) > 0);
		assertTrue(vertex.getStateTimestamp(ExecutionState.FAILED) > 0);
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

/**
 * A utility function that computes an "aggregated" state for the vertex.
 *
 * <p>This state is not used anywhere in the  coordination, but can be used for display
 * in dashboards to as a summary for how the particular parallel operation represented by
 * this ExecutionJobVertex is currently behaving.
 *
 * <p>For example, if at least one parallel task is failed, the aggregate state is failed.
 * If not, and at least one parallel task is cancelling (or cancelled), the aggregate state
 * is cancelling (or cancelled). If all tasks are finished, the aggregate state is finished,
 * and so on.
 *
 * @param verticesPerState The number of vertices in each state (indexed by the ordinal of
 *                         the ExecutionState values).
 * @param parallelism The parallelism of the ExecutionJobVertex
 *
 * @return The aggregate state of this ExecutionJobVertex.
 */
public static ExecutionState getAggregateJobVertexState(int[] verticesPerState, int parallelism) {
	if (verticesPerState == null || verticesPerState.length != ExecutionState.values().length) {
		throw new IllegalArgumentException("Must provide an array as large as there are execution states.");
	}

	if (verticesPerState[ExecutionState.FAILED.ordinal()] > 0) {
		return ExecutionState.FAILED;
	}
	if (verticesPerState[ExecutionState.CANCELING.ordinal()] > 0) {
		return ExecutionState.CANCELING;
	}
	else if (verticesPerState[ExecutionState.CANCELED.ordinal()] > 0) {
		return ExecutionState.CANCELED;
	}
	else if (verticesPerState[ExecutionState.RUNNING.ordinal()] > 0) {
		return ExecutionState.RUNNING;
	}
	else if (verticesPerState[ExecutionState.FINISHED.ordinal()] > 0) {
		return verticesPerState[ExecutionState.FINISHED.ordinal()] == parallelism ?
				ExecutionState.FINISHED : ExecutionState.RUNNING;
	}
	else {
		// all else collapses under created
		return ExecutionState.CREATED;
	}
}
 

void notifyExecutionChange(
		final Execution execution,
		final ExecutionState newExecutionState,
		final Throwable error) {

	if (!isLegacyScheduling()) {
		return;
	}

	// see what this means for us. currently, the first FAILED state means -> FAILED
	if (newExecutionState == ExecutionState.FAILED) {
		final Throwable ex = error != null ? error : new FlinkException("Unknown Error (missing cause)");

		// by filtering out late failure calls, we can save some work in
		// avoiding redundant local failover
		if (execution.getGlobalModVersion() == globalModVersion) {
			try {
				// fail all checkpoints which the failed task has not yet acknowledged
				if (checkpointCoordinator != null) {
					checkpointCoordinator.failUnacknowledgedPendingCheckpointsFor(execution.getAttemptId(), ex);
				}

				failoverStrategy.onTaskFailure(execution, ex);
			}
			catch (Throwable t) {
				// bug in the failover strategy - fall back to global failover
				LOG.warn("Error in failover strategy - falling back to global restart", t);
				failGlobal(ex);
			}
		}
	}
}
 

void notifyExecutionChange(
		final Execution execution,
		final ExecutionState newExecutionState,
		final Throwable error) {

	// see what this means for us. currently, the first FAILED state means -> FAILED
	if (newExecutionState == ExecutionState.FAILED) {
		final Throwable ex = error != null ? error : new FlinkException("Unknown Error (missing cause)");

		// by filtering out late failure calls, we can save some work in
		// avoiding redundant local failover
		if (execution.getGlobalModVersion() == globalModVersion) {
			try {
				// fail all checkpoints which the failed task has not yet acknowledged
				if (checkpointCoordinator != null) {
					checkpointCoordinator.failUnacknowledgedPendingCheckpointsFor(execution.getAttemptId(), ex);
				}

				failoverStrategy.onTaskFailure(execution, ex);
			}
			catch (Throwable t) {
				// bug in the failover strategy - fall back to global failover
				LOG.warn("Error in failover strategy - falling back to global restart", t);
				failGlobal(ex);
			}
		}
	}
}
 

/**
 * A utility function that computes an "aggregated" state for the vertex.
 *
 * <p>This state is not used anywhere in the  coordination, but can be used for display
 * in dashboards to as a summary for how the particular parallel operation represented by
 * this ExecutionJobVertex is currently behaving.
 *
 * <p>For example, if at least one parallel task is failed, the aggregate state is failed.
 * If not, and at least one parallel task is cancelling (or cancelled), the aggregate state
 * is cancelling (or cancelled). If all tasks are finished, the aggregate state is finished,
 * and so on.
 *
 * @param verticesPerState The number of vertices in each state (indexed by the ordinal of
 *                         the ExecutionState values).
 * @param parallelism The parallelism of the ExecutionJobVertex
 *
 * @return The aggregate state of this ExecutionJobVertex.
 */
public static ExecutionState getAggregateJobVertexState(int[] verticesPerState, int parallelism) {
	if (verticesPerState == null || verticesPerState.length != ExecutionState.values().length) {
		throw new IllegalArgumentException("Must provide an array as large as there are execution states.");
	}

	if (verticesPerState[ExecutionState.FAILED.ordinal()] > 0) {
		return ExecutionState.FAILED;
	}
	if (verticesPerState[ExecutionState.CANCELING.ordinal()] > 0) {
		return ExecutionState.CANCELING;
	}
	else if (verticesPerState[ExecutionState.CANCELED.ordinal()] > 0) {
		return ExecutionState.CANCELED;
	}
	else if (verticesPerState[ExecutionState.RUNNING.ordinal()] > 0) {
		return ExecutionState.RUNNING;
	}
	else if (verticesPerState[ExecutionState.FINISHED.ordinal()] > 0) {
		return verticesPerState[ExecutionState.FINISHED.ordinal()] == parallelism ?
				ExecutionState.FINISHED : ExecutionState.RUNNING;
	}
	else {
		// all else collapses under created
		return ExecutionState.CREATED;
	}
}
 

private void abortConsumptionOrIgnoreCheckResult(ResponseHandle responseHandle) {
	ExecutionState producerState = getProducerState(responseHandle);
	if (producerState == ExecutionState.CANCELING ||
		producerState == ExecutionState.CANCELED ||
		producerState == ExecutionState.FAILED) {

		// The producing execution has been canceled or failed. We
		// don't need to re-trigger the request since it cannot
		// succeed.
		if (LOG.isDebugEnabled()) {
			LOG.debug("Cancelling task {} after the producer of partition {} with attempt ID {} has entered state {}.",
				taskNameWithSubtask,
				resultPartitionId.getPartitionId(),
				resultPartitionId.getProducerId(),
				producerState);
		}

		responseHandle.cancelConsumption();
	} else {
		// Any other execution state is unexpected. Currently, only
		// state CREATED is left out of the checked states. If we
		// see a producer in this state, something went wrong with
		// scheduling in topological order.
		final String msg = String.format("Producer with attempt ID %s of partition %s in unexpected state %s.",
			resultPartitionId.getProducerId(),
			resultPartitionId.getPartitionId(),
			producerState);

		responseHandle.failConsumption(new IllegalStateException(msg));
	}
}
 

@Test
public void testDeployFailedAsynchronously() {
	try {
		final JobVertexID jid = new JobVertexID();
		final ExecutionJobVertex ejv = getExecutionVertex(jid);
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
			AkkaUtils.getDefaultTimeout());

		final LogicalSlot slot = new TestingLogicalSlotBuilder().setTaskManagerGateway(new SubmitFailingSimpleAckingTaskManagerGateway()).createTestingLogicalSlot();

		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());

		vertex.deployToSlot(slot);

		// wait until the state transition must be done
		for (int i = 0; i < 100; i++) {
			if (vertex.getExecutionState() == ExecutionState.FAILED && vertex.getFailureCause() != null) {
				break;
			} else {
				Thread.sleep(10);
			}
		}

		assertEquals(ExecutionState.FAILED, vertex.getExecutionState());
		assertNotNull(vertex.getFailureCause());
		assertTrue(vertex.getFailureCause().getMessage().contains(ERROR_MESSAGE));

		assertTrue(vertex.getStateTimestamp(ExecutionState.CREATED) > 0);
		assertTrue(vertex.getStateTimestamp(ExecutionState.DEPLOYING) > 0);
		assertTrue(vertex.getStateTimestamp(ExecutionState.FAILED) > 0);
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

/**
 * Verifies that {@link ExecutionGraph#updateState(TaskExecutionState)} updates the accumulators and metrics for an
 * execution that failed or was canceled.
 */
@Test
public void testAccumulatorsAndMetricsForwarding() throws Exception {
	final JobVertexID jid1 = new JobVertexID();
	final JobVertexID jid2 = new JobVertexID();

	JobVertex v1 = new JobVertex("v1", jid1);
	JobVertex v2 = new JobVertex("v2", jid2);

	Tuple2<ExecutionGraph, Map<ExecutionAttemptID, Execution>> graphAndExecutions = setupExecution(v1, 1, v2, 1);
	ExecutionGraph graph = graphAndExecutions.f0;

	// verify behavior for canceled executions
	Execution execution1 = graphAndExecutions.f1.values().iterator().next();

	IOMetrics ioMetrics = new IOMetrics(0, 0, 0, 0);
	Map<String, Accumulator<?, ?>> accumulators = new HashMap<>();
	accumulators.put("acc", new IntCounter(4));
	AccumulatorSnapshot accumulatorSnapshot = new AccumulatorSnapshot(graph.getJobID(), execution1.getAttemptId(), accumulators);

	TaskExecutionState state = new TaskExecutionState(graph.getJobID(), execution1.getAttemptId(), ExecutionState.CANCELED, null, accumulatorSnapshot, ioMetrics);

	graph.updateState(state);

	assertEquals(ioMetrics, execution1.getIOMetrics());
	assertNotNull(execution1.getUserAccumulators());
	assertEquals(4, execution1.getUserAccumulators().get("acc").getLocalValue());

	// verify behavior for failed executions
	Execution execution2 = graphAndExecutions.f1.values().iterator().next();

	IOMetrics ioMetrics2 = new IOMetrics(0, 0, 0, 0);
	Map<String, Accumulator<?, ?>> accumulators2 = new HashMap<>();
	accumulators2.put("acc", new IntCounter(8));
	AccumulatorSnapshot accumulatorSnapshot2 = new AccumulatorSnapshot(graph.getJobID(), execution2.getAttemptId(), accumulators2);

	TaskExecutionState state2 = new TaskExecutionState(graph.getJobID(), execution2.getAttemptId(), ExecutionState.FAILED, null, accumulatorSnapshot2, ioMetrics2);

	graph.updateState(state2);

	assertEquals(ioMetrics2, execution2.getIOMetrics());
	assertNotNull(execution2.getUserAccumulators());
	assertEquals(8, execution2.getUserAccumulators().get("acc").getLocalValue());
}
 

private void maybeHandleTaskFailure(final TaskExecutionState taskExecutionState, final ExecutionVertexID executionVertexId) {
	if (taskExecutionState.getExecutionState() == ExecutionState.FAILED) {
		final Throwable error = taskExecutionState.getError(userCodeLoader);
		handleTaskFailure(executionVertexId, error);
	}
}
 

/**
 * Checks whether the task has failed, is canceled, or is being canceled at the moment.
 * @return True is the task in state FAILED, CANCELING, or CANCELED, false otherwise.
 */
public boolean isCanceledOrFailed() {
	return executionState == ExecutionState.CANCELING ||
			executionState == ExecutionState.CANCELED ||
			executionState == ExecutionState.FAILED;
}
 

private static boolean isProducerFailedOrCanceled(ExecutionState producerState) {
	return producerState == ExecutionState.CANCELING ||
		producerState == ExecutionState.CANCELED ||
		producerState == ExecutionState.FAILED;
}
 

/**
 * Verifies that {@link ExecutionGraph#updateState(TaskExecutionState)} updates the accumulators and metrics for an
 * execution that failed or was canceled.
 */
@Test
public void testAccumulatorsAndMetricsForwarding() throws Exception {
	final JobVertexID jid1 = new JobVertexID();
	final JobVertexID jid2 = new JobVertexID();

	JobVertex v1 = new JobVertex("v1", jid1);
	JobVertex v2 = new JobVertex("v2", jid2);

	Tuple2<ExecutionGraph, Map<ExecutionAttemptID, Execution>> graphAndExecutions = setupExecution(v1, 1, v2, 1);
	ExecutionGraph graph = graphAndExecutions.f0;

	// verify behavior for canceled executions
	Execution execution1 = graphAndExecutions.f1.values().iterator().next();

	IOMetrics ioMetrics = new IOMetrics(0, 0, 0, 0);
	Map<String, Accumulator<?, ?>> accumulators = new HashMap<>();
	accumulators.put("acc", new IntCounter(4));
	AccumulatorSnapshot accumulatorSnapshot = new AccumulatorSnapshot(graph.getJobID(), execution1.getAttemptId(), accumulators);

	TaskExecutionState state = new TaskExecutionState(graph.getJobID(), execution1.getAttemptId(), ExecutionState.CANCELED, null, accumulatorSnapshot, ioMetrics);

	graph.updateState(state);

	assertEquals(ioMetrics, execution1.getIOMetrics());
	assertNotNull(execution1.getUserAccumulators());
	assertEquals(4, execution1.getUserAccumulators().get("acc").getLocalValue());

	// verify behavior for failed executions
	Execution execution2 = graphAndExecutions.f1.values().iterator().next();

	IOMetrics ioMetrics2 = new IOMetrics(0, 0, 0, 0);
	Map<String, Accumulator<?, ?>> accumulators2 = new HashMap<>();
	accumulators2.put("acc", new IntCounter(8));
	AccumulatorSnapshot accumulatorSnapshot2 = new AccumulatorSnapshot(graph.getJobID(), execution2.getAttemptId(), accumulators2);

	TaskExecutionState state2 = new TaskExecutionState(graph.getJobID(), execution2.getAttemptId(), ExecutionState.FAILED, null, accumulatorSnapshot2, ioMetrics2);

	graph.updateState(state2);

	assertEquals(ioMetrics2, execution2.getIOMetrics());
	assertNotNull(execution2.getUserAccumulators());
	assertEquals(8, execution2.getUserAccumulators().get("acc").getLocalValue());
}
 

/**
 * Checks whether the task has failed, is canceled, or is being canceled at the moment.
 * @return True is the task in state FAILED, CANCELING, or CANCELED, false otherwise.
 */
public boolean isCanceledOrFailed() {
	return executionState == ExecutionState.CANCELING ||
			executionState == ExecutionState.CANCELED ||
			executionState == ExecutionState.FAILED;
}
 

private static boolean isProducerFailedOrCanceled(ExecutionState producerState) {
	return producerState == ExecutionState.CANCELING ||
		producerState == ExecutionState.CANCELED ||
		producerState == ExecutionState.FAILED;
}
 

/**
 * Verifies that {@link ExecutionGraph#updateState(TaskExecutionState)} updates the accumulators and metrics for an
 * execution that failed or was canceled.
 */
@Test
public void testAccumulatorsAndMetricsForwarding() throws Exception {
	final JobVertexID jid1 = new JobVertexID();
	final JobVertexID jid2 = new JobVertexID();

	JobVertex v1 = new JobVertex("v1", jid1);
	JobVertex v2 = new JobVertex("v2", jid2);

	Tuple2<ExecutionGraph, Map<ExecutionAttemptID, Execution>> graphAndExecutions = setupExecution(v1, 1, v2, 1);
	ExecutionGraph graph = graphAndExecutions.f0;

	// verify behavior for canceled executions
	Execution execution1 = graphAndExecutions.f1.values().iterator().next();

	IOMetrics ioMetrics = new IOMetrics(0, 0, 0, 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0);
	Map<String, Accumulator<?, ?>> accumulators = new HashMap<>();
	accumulators.put("acc", new IntCounter(4));
	AccumulatorSnapshot accumulatorSnapshot = new AccumulatorSnapshot(graph.getJobID(), execution1.getAttemptId(), accumulators);

	TaskExecutionState state = new TaskExecutionState(graph.getJobID(), execution1.getAttemptId(), ExecutionState.CANCELED, null, accumulatorSnapshot, ioMetrics);

	graph.updateState(state);

	assertEquals(ioMetrics, execution1.getIOMetrics());
	assertNotNull(execution1.getUserAccumulators());
	assertEquals(4, execution1.getUserAccumulators().get("acc").getLocalValue());

	// verify behavior for failed executions
	Execution execution2 = graphAndExecutions.f1.values().iterator().next();

	IOMetrics ioMetrics2 = new IOMetrics(0, 0, 0, 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0);
	Map<String, Accumulator<?, ?>> accumulators2 = new HashMap<>();
	accumulators2.put("acc", new IntCounter(8));
	AccumulatorSnapshot accumulatorSnapshot2 = new AccumulatorSnapshot(graph.getJobID(), execution2.getAttemptId(), accumulators2);

	TaskExecutionState state2 = new TaskExecutionState(graph.getJobID(), execution2.getAttemptId(), ExecutionState.FAILED, null, accumulatorSnapshot2, ioMetrics2);

	graph.updateState(state2);

	assertEquals(ioMetrics2, execution2.getIOMetrics());
	assertNotNull(execution2.getUserAccumulators());
	assertEquals(8, execution2.getUserAccumulators().get("acc").getLocalValue());
}
 

/**
 * Answer to a partition state check issued after a failed partition request.
 */
@VisibleForTesting
void onPartitionStateUpdate(
		IntermediateDataSetID intermediateDataSetId,
		ResultPartitionID resultPartitionId,
		ExecutionState producerState) throws IOException, InterruptedException {

	if (executionState == ExecutionState.RUNNING) {
		final SingleInputGate inputGate = inputGatesById.get(intermediateDataSetId);

		if (inputGate != null) {
			if (producerState == ExecutionState.SCHEDULED
				|| producerState == ExecutionState.DEPLOYING
				|| producerState == ExecutionState.RUNNING
				|| producerState == ExecutionState.FINISHED) {

				// Retrigger the partition request
				inputGate.retriggerPartitionRequest(resultPartitionId.getPartitionId());

			} else if (producerState == ExecutionState.CANCELING
				|| producerState == ExecutionState.CANCELED
				|| producerState == ExecutionState.FAILED) {

				// The producing execution has been canceled or failed. We
				// don't need to re-trigger the request since it cannot
				// succeed.
				if (LOG.isDebugEnabled()) {
					LOG.debug("Cancelling task {} after the producer of partition {} with attempt ID {} has entered state {}.",
						taskNameWithSubtask,
						resultPartitionId.getPartitionId(),
						resultPartitionId.getProducerId(),
						producerState);
				}

				cancelExecution();
			} else {
				// Any other execution state is unexpected. Currently, only
				// state CREATED is left out of the checked states. If we
				// see a producer in this state, something went wrong with
				// scheduling in topological order.
				String msg = String.format("Producer with attempt ID %s of partition %s in unexpected state %s.",
					resultPartitionId.getProducerId(),
					resultPartitionId.getPartitionId(),
					producerState);

				failExternally(new IllegalStateException(msg));
			}
		} else {
			failExternally(new IllegalStateException("Received partition producer state for " +
					"unknown input gate " + intermediateDataSetId + "."));
		}
	} else {
		LOG.debug("Task {} ignored a partition producer state notification, because it's not running.", taskNameWithSubtask);
	}
}