org.apache.flink.runtime.io.network.partition.consumer.StreamTestSingleInputGate Java Examples

@Override
protected void initializeInputs() {
	inputGates = new StreamTestSingleInputGate[numInputGates];

	for (int i = 0; i < numInputGates; i++) {
		inputGates[i] = new StreamTestSingleInputGate<IN>(
			numInputChannelsPerGate,
			i,
			inputSerializer,
			bufferSize
		);
		this.mockEnv.addInputGate(inputGates[i].getInputGate());
	}

	streamConfig.setNumberOfInputs(1);
	streamConfig.setTypeSerializersIn(inputSerializer);
}
 

@Override
protected void initializeInputs() throws IOException, InterruptedException {
	inputGates = new StreamTestSingleInputGate[numInputGates];

	for (int i = 0; i < numInputGates; i++) {
		inputGates[i] = new StreamTestSingleInputGate<IN>(
			numInputChannelsPerGate,
			bufferSize,
			inputSerializer);
		this.mockEnv.addInputGate(inputGates[i].getInputGate());
	}

	streamConfig.setNumberOfInputs(1);
	streamConfig.setTypeSerializerIn1(inputSerializer);
}
 

@Override
protected void initializeInputs() throws IOException, InterruptedException {
	inputGates = new StreamTestSingleInputGate[numInputGates];

	for (int i = 0; i < numInputGates; i++) {
		inputGates[i] = new StreamTestSingleInputGate<IN>(
			numInputChannelsPerGate,
			bufferSize,
			inputSerializer);
		this.mockEnv.addInputGate(inputGates[i].getInputGate());
	}

	streamConfig.setNumberOfInputs(1);
	streamConfig.setTypeSerializerIn1(inputSerializer);
}
 

@Override
protected void initializeInputs(StreamMockEnvironment streamMockEnvironment) {
	inputGates = new StreamTestSingleInputGate[inputSerializers.size()];
	List<StreamEdge> inPhysicalEdges = new LinkedList<>();

	StreamOperator<?> dummyOperator = new AbstractStreamOperator<Object>() {
		private static final long serialVersionUID = 1L;
	};

	StreamNode sourceVertexDummy = new StreamNode(0, "default group", null, dummyOperator, "source dummy", new LinkedList<>(), SourceStreamTask.class);
	StreamNode targetVertexDummy = new StreamNode(1, "default group", null, dummyOperator, "target dummy", new LinkedList<>(), SourceStreamTask.class);

	for (int i = 0; i < inputSerializers.size(); i++) {
		TypeSerializer<?> inputSerializer = inputSerializers.get(i);
		inputGates[i] = new StreamTestSingleInputGate<>(
			inputChannelsPerGate.get(i),
			i,
			inputSerializer,
			bufferSize);

		StreamEdge streamEdge = new StreamEdge(
			sourceVertexDummy,
			targetVertexDummy,
			i + 1,
			new LinkedList<>(),
			new BroadcastPartitioner<>(),
			null);

		inPhysicalEdges.add(streamEdge);
		streamMockEnvironment.addInputGate(inputGates[i].getInputGate());
	}

	streamConfig.setInPhysicalEdges(inPhysicalEdges);
	streamConfig.setNumberOfInputs(inputGates.length);
	streamConfig.setTypeSerializersIn(inputSerializers.toArray(new TypeSerializer[inputSerializers.size()]));
}
 

StreamTaskMailboxTestHarness(
		StreamTask<OUT, ?> streamTask,
		Queue<Object> outputList,
		StreamTestSingleInputGate[] inputGates,
		StreamMockEnvironment streamMockEnvironment) {
	this.streamTask = checkNotNull(streamTask);
	this.taskStateManager = (TestTaskStateManager) streamMockEnvironment.getTaskStateManager();
	this.inputGates = checkNotNull(inputGates);
	this.outputList = checkNotNull(outputList);
	this.streamMockEnvironment = checkNotNull(streamMockEnvironment);
	this.inputGateEnded = new boolean[inputGates.length];
}
 

@Test
public void testReleasingDeserializerTimely()
	throws Exception {

	int numInputChannels = 2;
	LongSerializer inSerializer = LongSerializer.INSTANCE;
	StreamTestSingleInputGate inputGate = new StreamTestSingleInputGate<>(numInputChannels, 0, inSerializer, 1024);

	TestRecordDeserializer[] deserializers = new TestRecordDeserializer[numInputChannels];
	for (int i = 0; i < deserializers.length; i++) {
		deserializers[i] = new TestRecordDeserializer(ioManager.getSpillingDirectoriesPaths());
	}

	TestRecordDeserializer[] copiedDeserializers = Arrays.copyOf(deserializers, deserializers.length);
	DataOutput output = new NoOpDataOutput<>();
	StreamTaskNetworkInput input = new StreamTaskNetworkInput<>(
		new CheckpointedInputGate(
			inputGate.getInputGate(),
			new CheckpointBarrierTracker(1, new DummyCheckpointInvokable())),
		inSerializer,
		new StatusWatermarkValve(1, output),
		0,
		deserializers);

	for (int i = 0; i < numInputChannels; i++) {
		assertNotNull(deserializers[i]);
		inputGate.sendEvent(EndOfPartitionEvent.INSTANCE, i);
		input.emitNext(output);
		assertNull(deserializers[i]);
		assertTrue(copiedDeserializers[i].isCleared());
	}
}
 

@Test
public void testSnapshotAfterEndOfPartition() throws Exception {
	int numInputChannels = 1;
	int channelId = 0;
	int checkpointId = 0;

	VerifyRecordsDataOutput<Long> output = new VerifyRecordsDataOutput<>();
	LongSerializer inSerializer = LongSerializer.INSTANCE;
	StreamTestSingleInputGate<Long> inputGate = new StreamTestSingleInputGate<>(numInputChannels, 0, inSerializer, 1024);
	TestRecordDeserializer[] deserializers = createDeserializers(numInputChannels);
	StreamTaskNetworkInput<Long> input = new StreamTaskNetworkInput<>(
		new CheckpointedInputGate(
			inputGate.getInputGate(),
			new CheckpointBarrierUnaligner(
				TestSubtaskCheckpointCoordinator.INSTANCE,
				"test",
				new DummyCheckpointInvokable(),
				inputGate.getInputGate())),
		inSerializer,
		new StatusWatermarkValve(numInputChannels, output),
		0,
		deserializers);

	inputGate.sendEvent(
		new CheckpointBarrier(checkpointId, 0L, CheckpointOptions.forCheckpointWithDefaultLocation()),
		channelId);
	inputGate.sendElement(new StreamRecord<>(42L), channelId);

	assertHasNextElement(input, output);
	assertHasNextElement(input, output);
	assertEquals(1, output.getNumberOfEmittedRecords());

	// send EndOfPartitionEvent and ensure that deserializer has been released
	inputGate.sendEvent(EndOfPartitionEvent.INSTANCE, channelId);
	input.emitNext(output);
	assertNull(deserializers[channelId]);

	// now snapshot all inflight buffers
	CompletableFuture<Void> completableFuture = input.prepareSnapshot(ChannelStateWriter.NO_OP, checkpointId);
	completableFuture.join();
}