org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID Java Examples
The following examples show how to use
org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ExecutionGraph.java From flink with Apache License 2.0 | 6 votes |
ResultPartitionID createResultPartitionId(final IntermediateResultPartitionID resultPartitionId) { final SchedulingResultPartition schedulingResultPartition = schedulingTopology.getResultPartitionOrThrow(resultPartitionId); final SchedulingExecutionVertex producer = schedulingResultPartition.getProducer(); final ExecutionVertexID producerId = producer.getId(); final JobVertexID jobVertexId = producerId.getJobVertexId(); final ExecutionJobVertex jobVertex = getJobVertex(jobVertexId); checkNotNull(jobVertex, "Unknown job vertex %s", jobVertexId); final ExecutionVertex[] taskVertices = jobVertex.getTaskVertices(); final int subtaskIndex = producerId.getSubtaskIndex(); checkState(subtaskIndex < taskVertices.length, "Invalid subtask index %d for job vertex %s", subtaskIndex, jobVertexId); final ExecutionVertex taskVertex = taskVertices[subtaskIndex]; final Execution execution = taskVertex.getCurrentExecutionAttempt(); return new ResultPartitionID(resultPartitionId, execution.getAttemptId()); }
Example #2
Source File: ExecutionGraph.java From flink with Apache License 2.0 | 6 votes |
ResultPartitionID createResultPartitionId(final IntermediateResultPartitionID resultPartitionId) { final SchedulingResultPartition schedulingResultPartition = getSchedulingTopology().getResultPartition(resultPartitionId); final SchedulingExecutionVertex producer = schedulingResultPartition.getProducer(); final ExecutionVertexID producerId = producer.getId(); final JobVertexID jobVertexId = producerId.getJobVertexId(); final ExecutionJobVertex jobVertex = getJobVertex(jobVertexId); checkNotNull(jobVertex, "Unknown job vertex %s", jobVertexId); final ExecutionVertex[] taskVertices = jobVertex.getTaskVertices(); final int subtaskIndex = producerId.getSubtaskIndex(); checkState(subtaskIndex < taskVertices.length, "Invalid subtask index %d for job vertex %s", subtaskIndex, jobVertexId); final ExecutionVertex taskVertex = taskVertices[subtaskIndex]; final Execution execution = taskVertex.getCurrentExecutionAttempt(); return new ResultPartitionID(resultPartitionId, execution.getAttemptId()); }
Example #3
Source File: SingleInputGateTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test(timeout = 120 * 1000) public void testIsMoreAvailableReadingFromSingleInputChannel() throws Exception { // Setup final SingleInputGate inputGate = createInputGate(); final TestInputChannel[] inputChannels = new TestInputChannel[]{ new TestInputChannel(inputGate, 0), new TestInputChannel(inputGate, 1) }; inputGate.setInputChannel( new IntermediateResultPartitionID(), inputChannels[0]); inputGate.setInputChannel( new IntermediateResultPartitionID(), inputChannels[1]); // Test inputChannels[0].readBuffer(); inputChannels[0].readBuffer(false); inputGate.notifyChannelNonEmpty(inputChannels[0]); verifyBufferOrEvent(inputGate, true, 0, true); verifyBufferOrEvent(inputGate, true, 0, false); }
Example #4
Source File: ResultPartitionManager.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
public void releasePartitionsProducedBy(ExecutionAttemptID executionId, Throwable cause) { synchronized (registeredPartitions) { final Map<IntermediateResultPartitionID, ResultPartition> partitions = registeredPartitions.row(executionId); for (ResultPartition partition : partitions.values()) { partition.release(cause); } for (IntermediateResultPartitionID partitionId : ImmutableList .copyOf(partitions.keySet())) { registeredPartitions.remove(executionId, partitionId); } LOG.debug("Released all partitions produced by {}.", executionId); } }
Example #5
Source File: RegionPartitionReleaseStrategyTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void releasePartitionsIfDownstreamRegionWithMultipleOperatorsIsFinished() { final List<TestingSchedulingExecutionVertex> sourceVertices = testingSchedulingTopology.addExecutionVertices().finish(); final List<TestingSchedulingExecutionVertex> intermediateVertices = testingSchedulingTopology.addExecutionVertices().finish(); final List<TestingSchedulingExecutionVertex> sinkVertices = testingSchedulingTopology.addExecutionVertices().finish(); final List<TestingSchedulingResultPartition> sourceResultPartitions = testingSchedulingTopology.connectAllToAll(sourceVertices, intermediateVertices).finish(); testingSchedulingTopology.connectAllToAll(intermediateVertices, sinkVertices).withResultPartitionType(ResultPartitionType.PIPELINED).finish(); final ExecutionVertexID onlyIntermediateVertexId = intermediateVertices.get(0).getId(); final ExecutionVertexID onlySinkVertexId = sinkVertices.get(0).getId(); final IntermediateResultPartitionID onlySourceResultPartitionId = sourceResultPartitions.get(0).getId(); final RegionPartitionReleaseStrategy regionPartitionReleaseStrategy = new RegionPartitionReleaseStrategy(testingSchedulingTopology); regionPartitionReleaseStrategy.vertexFinished(onlyIntermediateVertexId); final List<IntermediateResultPartitionID> partitionsToRelease = regionPartitionReleaseStrategy.vertexFinished(onlySinkVertexId); assertThat(partitionsToRelease, contains(onlySourceResultPartitionId)); }
Example #6
Source File: NettyMessage.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
static TaskEventRequest readFrom(ByteBuf buffer, ClassLoader classLoader) throws IOException { // directly deserialize fromNetty's buffer int length = buffer.readInt(); ByteBuffer serializedEvent = buffer.nioBuffer(buffer.readerIndex(), length); // assume this event's content is read from the ByteBuf (positions are not shared!) buffer.readerIndex(buffer.readerIndex() + length); TaskEvent event = (TaskEvent) EventSerializer.fromSerializedEvent(serializedEvent, classLoader); ResultPartitionID partitionId = new ResultPartitionID( IntermediateResultPartitionID.fromByteBuf(buffer), ExecutionAttemptID.fromByteBuf(buffer)); InputChannelID receiverId = InputChannelID.fromByteBuf(buffer); return new TaskEventRequest(event, partitionId, receiverId); }
Example #7
Source File: ResultPartitionDeploymentDescriptor.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
public ResultPartitionDeploymentDescriptor( IntermediateDataSetID resultId, IntermediateResultPartitionID partitionId, ResultPartitionType partitionType, int numberOfSubpartitions, int maxParallelism, boolean lazyScheduling) { this.resultId = checkNotNull(resultId); this.partitionId = checkNotNull(partitionId); this.partitionType = checkNotNull(partitionType); KeyGroupRangeAssignment.checkParallelismPreconditions(maxParallelism); checkArgument(numberOfSubpartitions >= 1); this.numberOfSubpartitions = numberOfSubpartitions; this.maxParallelism = maxParallelism; this.sendScheduleOrUpdateConsumersMessage = lazyScheduling; }
Example #8
Source File: RegionPartitionReleaseStrategyTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void toggleVertexFinishedUnfinished() { final List<TestingSchedulingExecutionVertex> producers = testingSchedulingTopology.addExecutionVertices().finish(); final List<TestingSchedulingExecutionVertex> consumers = testingSchedulingTopology.addExecutionVertices().withParallelism(2).finish(); testingSchedulingTopology.connectAllToAll(producers, consumers).finish(); final ExecutionVertexID consumerVertex1 = consumers.get(0).getId(); final ExecutionVertexID consumerVertex2 = consumers.get(1).getId(); final RegionPartitionReleaseStrategy regionPartitionReleaseStrategy = new RegionPartitionReleaseStrategy(testingSchedulingTopology); regionPartitionReleaseStrategy.vertexFinished(consumerVertex1); regionPartitionReleaseStrategy.vertexFinished(consumerVertex2); regionPartitionReleaseStrategy.vertexUnfinished(consumerVertex2); final List<IntermediateResultPartitionID> partitionsToRelease = regionPartitionReleaseStrategy.vertexFinished(consumerVertex1); assertThat(partitionsToRelease, is(empty())); }
Example #9
Source File: NettyMessage.java From flink with Apache License 2.0 | 6 votes |
static TaskEventRequest readFrom(ByteBuf buffer, ClassLoader classLoader) throws IOException { // directly deserialize fromNetty's buffer int length = buffer.readInt(); ByteBuffer serializedEvent = buffer.nioBuffer(buffer.readerIndex(), length); // assume this event's content is read from the ByteBuf (positions are not shared!) buffer.readerIndex(buffer.readerIndex() + length); TaskEvent event = (TaskEvent) EventSerializer.fromSerializedEvent(serializedEvent, classLoader); ResultPartitionID partitionId = new ResultPartitionID( IntermediateResultPartitionID.fromByteBuf(buffer), ExecutionAttemptID.fromByteBuf(buffer)); InputChannelID receiverId = InputChannelID.fromByteBuf(buffer); return new TaskEventRequest(event, partitionId, receiverId); }
Example #10
Source File: RegionPartitionReleaseStrategy.java From flink with Apache License 2.0 | 5 votes |
@Override public List<IntermediateResultPartitionID> vertexFinished(final ExecutionVertexID finishedVertex) { final PipelinedRegionExecutionView regionExecutionView = getPipelinedRegionExecutionViewForVertex(finishedVertex); regionExecutionView.vertexFinished(finishedVertex); if (regionExecutionView.isFinished()) { final PipelinedRegion pipelinedRegion = getPipelinedRegionForVertex(finishedVertex); final PipelinedRegionConsumedBlockingPartitions consumedPartitionsOfVertexRegion = getConsumedBlockingPartitionsForRegion(pipelinedRegion); return filterReleasablePartitions(consumedPartitionsOfVertexRegion); } return Collections.emptyList(); }
Example #11
Source File: DefaultExecutionTopologyTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testResultPartitionOrThrow() { try { adapter.getResultPartition(new IntermediateResultPartitionID()); fail("get not exist result partition"); } catch (IllegalArgumentException exception) { // expected } }
Example #12
Source File: RegionPartitionReleaseStrategy.java From flink with Apache License 2.0 | 5 votes |
private List<IntermediateResultPartitionID> filterReleasablePartitions(final PipelinedRegionConsumedBlockingPartitions consumedPartitionsOfVertexRegion) { return consumedPartitionsOfVertexRegion .getConsumedBlockingPartitions() .stream() .filter(this::areConsumerRegionsFinished) .collect(Collectors.toList()); }
Example #13
Source File: TestingSchedulingTopology.java From flink with Apache License 2.0 | 5 votes |
@Override public TestingSchedulingResultPartition getResultPartition(final IntermediateResultPartitionID intermediateResultPartitionId) { final TestingSchedulingResultPartition resultPartition = schedulingResultPartitions.get(intermediateResultPartitionId); if (resultPartition == null) { throw new IllegalArgumentException("can not find partition: " + intermediateResultPartitionId); } return resultPartition; }
Example #14
Source File: ExecutionGraphResultPartitionAvailabilityCheckerTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPartitionAvailabilityCheck() { final IntermediateResultPartitionID irp1ID = new IntermediateResultPartitionID(); final IntermediateResultPartitionID irp2ID = new IntermediateResultPartitionID(); final IntermediateResultPartitionID irp3ID = new IntermediateResultPartitionID(); final IntermediateResultPartitionID irp4ID = new IntermediateResultPartitionID(); final Map<IntermediateResultPartitionID, Boolean> expectedAvailability = new HashMap<IntermediateResultPartitionID, Boolean>() {{ put(irp1ID, true); put(irp2ID, false); put(irp3ID, false); put(irp4ID, true); }}; // let the partition tracker respect the expected availability result final TestingJobMasterPartitionTracker partitionTracker = new TestingJobMasterPartitionTracker(); partitionTracker.setIsPartitionTrackedFunction(rpID -> expectedAvailability.get(rpID.getPartitionId())); // the execution attempt ID should make no difference in this case final Function<IntermediateResultPartitionID, ResultPartitionID> partitionIDMapper = intermediateResultPartitionID -> new ResultPartitionID(intermediateResultPartitionID, new ExecutionAttemptID()); final ResultPartitionAvailabilityChecker resultPartitionAvailabilityChecker = new ExecutionGraphResultPartitionAvailabilityChecker(partitionIDMapper, partitionTracker); for (IntermediateResultPartitionID irpID : expectedAvailability.keySet()) { assertEquals(expectedAvailability.get(irpID), resultPartitionAvailabilityChecker.isAvailable(irpID)); } }
Example #15
Source File: SingleInputGateTest.java From flink with Apache License 2.0 | 5 votes |
private static Map<InputGateID, SingleInputGate> createInputGateWithLocalChannels( NettyShuffleEnvironment network, int numberOfGates, @SuppressWarnings("SameParameterValue") int numberOfLocalChannels) { ShuffleDescriptor[] channelDescs = new NettyShuffleDescriptor[numberOfLocalChannels]; for (int i = 0; i < numberOfLocalChannels; i++) { channelDescs[i] = createRemoteWithIdAndLocation(new IntermediateResultPartitionID(), ResourceID.generate()); } InputGateDeploymentDescriptor[] gateDescs = new InputGateDeploymentDescriptor[numberOfGates]; IntermediateDataSetID[] ids = new IntermediateDataSetID[numberOfGates]; for (int i = 0; i < numberOfGates; i++) { ids[i] = new IntermediateDataSetID(); gateDescs[i] = new InputGateDeploymentDescriptor( ids[i], ResultPartitionType.PIPELINED, 0, channelDescs); } ExecutionAttemptID consumerID = new ExecutionAttemptID(); SingleInputGate[] gates = network.createInputGates( network.createShuffleIOOwnerContext("", consumerID, new UnregisteredMetricsGroup()), SingleInputGateBuilder.NO_OP_PRODUCER_CHECKER, Arrays.asList(gateDescs)).toArray(new SingleInputGate[] {}); Map<InputGateID, SingleInputGate> inputGatesById = new HashMap<>(); for (int i = 0; i < numberOfGates; i++) { inputGatesById.put(new InputGateID(ids[i], consumerID), gates[i]); } return inputGatesById; }
Example #16
Source File: IteratorWrappingTestSingleInputGate.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private IteratorWrappingTestSingleInputGate<T> wrapIterator(MutableObjectIterator<T> iterator) throws IOException, InterruptedException { inputIterator = iterator; serializer = new SpanningRecordSerializer<T>(); // The input iterator can produce an infinite stream. That's why we have to serialize each // record on demand and cannot do it upfront. final BufferAndAvailabilityProvider answer = new BufferAndAvailabilityProvider() { private boolean hasData = inputIterator.next(reuse) != null; @Override public Optional<BufferAndAvailability> getBufferAvailability() throws IOException { if (hasData) { serializer.serializeRecord(reuse); BufferBuilder bufferBuilder = createBufferBuilder(bufferSize); serializer.copyToBufferBuilder(bufferBuilder); hasData = inputIterator.next(reuse) != null; // Call getCurrentBuffer to ensure size is set return Optional.of(new BufferAndAvailability(buildSingleBuffer(bufferBuilder), true, 0)); } else { inputChannel.setReleased(); return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE), false, 0)); } } }; inputChannel.addBufferAndAvailability(answer); inputGate.setInputChannel(new IntermediateResultPartitionID(), inputChannel); return this; }
Example #17
Source File: ResultPartitionDeploymentDescriptorTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Tests simple de/serialization. */ @Test public void testSerialization() throws Exception { // Expected values IntermediateDataSetID resultId = new IntermediateDataSetID(); IntermediateResultPartitionID partitionId = new IntermediateResultPartitionID(); ResultPartitionType partitionType = ResultPartitionType.PIPELINED; int numberOfSubpartitions = 24; ResultPartitionDeploymentDescriptor orig = new ResultPartitionDeploymentDescriptor( resultId, partitionId, partitionType, numberOfSubpartitions, numberOfSubpartitions, true); ResultPartitionDeploymentDescriptor copy = CommonTestUtils.createCopySerializable(orig); assertEquals(resultId, copy.getResultId()); assertEquals(partitionId, copy.getPartitionId()); assertEquals(partitionType, copy.getPartitionType()); assertEquals(numberOfSubpartitions, copy.getNumberOfSubpartitions()); assertTrue(copy.sendScheduleOrUpdateConsumersMessage()); }
Example #18
Source File: RegionPartitionReleaseStrategyTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void notReleasePartitionsIfDownstreamRegionIsNotFinished() { final List<TestingSchedulingExecutionVertex> producers = testingSchedulingTopology.addExecutionVertices().finish(); final List<TestingSchedulingExecutionVertex> consumers = testingSchedulingTopology.addExecutionVertices().withParallelism(2).finish(); testingSchedulingTopology.connectAllToAll(producers, consumers).finish(); final ExecutionVertexID consumerVertex1 = consumers.get(0).getId(); final RegionPartitionReleaseStrategy regionPartitionReleaseStrategy = new RegionPartitionReleaseStrategy(testingSchedulingTopology); final List<IntermediateResultPartitionID> partitionsToRelease = regionPartitionReleaseStrategy.vertexFinished(consumerVertex1); assertThat(partitionsToRelease, is(empty())); }
Example #19
Source File: IntermediateResultPartition.java From flink with Apache License 2.0 | 5 votes |
public IntermediateResultPartition(IntermediateResult totalResult, ExecutionVertex producer, int partitionNumber) { this.totalResult = totalResult; this.producer = producer; this.partitionNumber = partitionNumber; this.consumers = new ArrayList<List<ExecutionEdge>>(0); this.partitionId = new IntermediateResultPartitionID(totalResult.getId(), partitionNumber); }
Example #20
Source File: SingleInputGateTest.java From flink with Apache License 2.0 | 5 votes |
private static Map<InputGateID, SingleInputGate> createInputGateWithLocalChannels( NettyShuffleEnvironment network, int numberOfGates, @SuppressWarnings("SameParameterValue") int numberOfLocalChannels) { ShuffleDescriptor[] channelDescs = new NettyShuffleDescriptor[numberOfLocalChannels]; for (int i = 0; i < numberOfLocalChannels; i++) { channelDescs[i] = createRemoteWithIdAndLocation(new IntermediateResultPartitionID(), ResourceID.generate()); } InputGateDeploymentDescriptor[] gateDescs = new InputGateDeploymentDescriptor[numberOfGates]; IntermediateDataSetID[] ids = new IntermediateDataSetID[numberOfGates]; for (int i = 0; i < numberOfGates; i++) { ids[i] = new IntermediateDataSetID(); gateDescs[i] = new InputGateDeploymentDescriptor( ids[i], ResultPartitionType.PIPELINED, 0, channelDescs); } ExecutionAttemptID consumerID = new ExecutionAttemptID(); SingleInputGate[] gates = network.createInputGates( network.createShuffleIOOwnerContext("", consumerID, new UnregisteredMetricsGroup()), SingleInputGateBuilder.NO_OP_PRODUCER_CHECKER, asList(gateDescs)).toArray(new SingleInputGate[] {}); Map<InputGateID, SingleInputGate> inputGatesById = new HashMap<>(); for (int i = 0; i < numberOfGates; i++) { inputGatesById.put(new InputGateID(ids[i], consumerID), gates[i]); } return inputGatesById; }
Example #21
Source File: ExecutionGraphToSchedulingTopologyAdapter.java From flink with Apache License 2.0 | 5 votes |
private static List<DefaultSchedulingResultPartition> generateProducedSchedulingResultPartition( Map<IntermediateResultPartitionID, IntermediateResultPartition> producedIntermediatePartitions) { List<DefaultSchedulingResultPartition> producedSchedulingPartitions = new ArrayList<>(producedIntermediatePartitions.size()); producedIntermediatePartitions.values().forEach( irp -> producedSchedulingPartitions.add( new DefaultSchedulingResultPartition( irp.getPartitionId(), irp.getIntermediateResult().getId(), irp.getResultType()))); return producedSchedulingPartitions; }
Example #22
Source File: DefaultExecutionTopology.java From flink with Apache License 2.0 | 5 votes |
@Override public DefaultResultPartition getResultPartition(final IntermediateResultPartitionID intermediateResultPartitionId) { final DefaultResultPartition resultPartition = resultPartitionsById.get(intermediateResultPartitionId); if (resultPartition == null) { throw new IllegalArgumentException("can not find partition: " + intermediateResultPartitionId); } return resultPartition; }
Example #23
Source File: ExecutionGraphResultPartitionAvailabilityChecker.java From flink with Apache License 2.0 | 5 votes |
ExecutionGraphResultPartitionAvailabilityChecker( final Function<IntermediateResultPartitionID, ResultPartitionID> partitionIDMapper, final PartitionTracker partitionTracker) { this.partitionIDMapper = checkNotNull(partitionIDMapper); this.partitionTracker = checkNotNull(partitionTracker); }
Example #24
Source File: NettyShuffleDescriptorBuilder.java From flink with Apache License 2.0 | 5 votes |
public static NettyShuffleDescriptor createRemoteWithIdAndLocation( IntermediateResultPartitionID partitionId, ResourceID producerLocation) { return newBuilder() .setId(new ResultPartitionID(partitionId, new ExecutionAttemptID())) .setProducerLocation(producerLocation) .buildRemote(); }
Example #25
Source File: Execution.java From flink with Apache License 2.0 | 5 votes |
@VisibleForTesting static CompletableFuture<Map<IntermediateResultPartitionID, ResultPartitionDeploymentDescriptor>> registerProducedPartitions( ExecutionVertex vertex, TaskManagerLocation location, ExecutionAttemptID attemptId) { ProducerDescriptor producerDescriptor = ProducerDescriptor.create(location, attemptId); boolean lazyScheduling = vertex.getExecutionGraph().getScheduleMode().allowLazyDeployment(); Collection<IntermediateResultPartition> partitions = vertex.getProducedPartitions().values(); Collection<CompletableFuture<ResultPartitionDeploymentDescriptor>> partitionRegistrations = new ArrayList<>(partitions.size()); for (IntermediateResultPartition partition : partitions) { PartitionDescriptor partitionDescriptor = PartitionDescriptor.from(partition); int maxParallelism = getPartitionMaxParallelism(partition); CompletableFuture<? extends ShuffleDescriptor> shuffleDescriptorFuture = vertex .getExecutionGraph() .getShuffleMaster() .registerPartitionWithProducer(partitionDescriptor, producerDescriptor); CompletableFuture<ResultPartitionDeploymentDescriptor> partitionRegistration = shuffleDescriptorFuture .thenApply(shuffleDescriptor -> new ResultPartitionDeploymentDescriptor( partitionDescriptor, shuffleDescriptor, maxParallelism, lazyScheduling)); partitionRegistrations.add(partitionRegistration); } return FutureUtils.combineAll(partitionRegistrations).thenApply(rpdds -> { Map<IntermediateResultPartitionID, ResultPartitionDeploymentDescriptor> producedPartitions = new LinkedHashMap<>(partitions.size()); rpdds.forEach(rpdd -> producedPartitions.put(rpdd.getPartitionId(), rpdd)); return producedPartitions; }); }
Example #26
Source File: DefaultExecutionVertexTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testGetConsumedResultPartitions() { IntermediateResultPartitionID partitionIds1 = IterableUtils .toStream(consumerVertex.getConsumedResults()) .findAny() .map(SchedulingResultPartition::getId) .orElseThrow(() -> new IllegalArgumentException("can not find result partition")); assertEquals(partitionIds1, intermediateResultPartitionId); }
Example #27
Source File: IntermediateResult.java From flink with Apache License 2.0 | 5 votes |
/** * Returns the partition with the given ID. * * @param resultPartitionId ID of the partition to look up * @throws NullPointerException If partition ID <code>null</code> * @throws IllegalArgumentException Thrown if unknown partition ID * @return Intermediate result partition with the given ID */ public IntermediateResultPartition getPartitionById(IntermediateResultPartitionID resultPartitionId) { // Looks ups the partition number via the helper map and returns the // partition. Currently, this happens infrequently enough that we could // consider removing the map and scanning the partitions on every lookup. // The lookup (currently) only happen when the producer of an intermediate // result cannot be found via its registered execution. Integer partitionNumber = partitionLookupHelper.get(checkNotNull(resultPartitionId, "IntermediateResultPartitionID")); if (partitionNumber != null) { return partitions[partitionNumber]; } else { throw new IllegalArgumentException("Unknown intermediate result partition ID " + resultPartitionId); } }
Example #28
Source File: ExecutionGraph.java From flink with Apache License 2.0 | 5 votes |
private void maybeReleasePartitions(final Execution attempt) { final ExecutionVertexID finishedExecutionVertex = attempt.getVertex().getID(); if (attempt.getState() == ExecutionState.FINISHED) { final List<IntermediateResultPartitionID> releasablePartitions = partitionReleaseStrategy.vertexFinished(finishedExecutionVertex); releasePartitions(releasablePartitions); } else { partitionReleaseStrategy.vertexUnfinished(finishedExecutionVertex); } }
Example #29
Source File: TestingSchedulingResultPartition.java From flink with Apache License 2.0 | 5 votes |
TestingSchedulingResultPartition(IntermediateDataSetID dataSetID, ResultPartitionType type, ResultPartitionState state) { this.intermediateDataSetID = dataSetID; this.partitionType = type; this.state = state; this.intermediateResultPartitionID = new IntermediateResultPartitionID(); this.consumers = new ArrayList<>(); }
Example #30
Source File: DefaultSchedulingExecutionVertexTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testGetConsumedResultPartitions() { IntermediateResultPartitionID partitionIds1 = consumerVertex .getConsumedResultPartitions().stream().findAny().map(SchedulingResultPartition::getId) .orElseThrow(() -> new IllegalArgumentException("can not find result partition")); assertEquals(partitionIds1, intermediateResultPartitionId); }