Java Code Examples for org.apache.flink.runtime.io.network.util.TestBufferFactory#createBuffer()

@Test
public void testExceptionOnReordering() throws Exception {
	// Setup
	final SingleInputGate inputGate = createSingleInputGate(1);
	final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);
	final Buffer buffer = TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE);

	// The test
	inputChannel.onBuffer(buffer.retainBuffer(), 0, -1);

	// This does not yet throw the exception, but sets the error at the channel.
	inputChannel.onBuffer(buffer, 29, -1);

	try {
		inputChannel.getNextBuffer();

		fail("Did not throw expected exception after enqueuing an out-of-order buffer.");
	}
	catch (Exception expected) {
		assertFalse(buffer.isRecycled());
		// free remaining buffer instances
		inputChannel.releaseAllResources();
		assertTrue(buffer.isRecycled());
	}
}
 

@Nullable
@Override
public BufferAndBacklog getNextBuffer() {
	int buffers = buffersInBacklog.decrementAndGet();
	return new BufferAndBacklog(
		TestBufferFactory.createBuffer(10),
		buffers > 0,
		buffers,
		false);
}
 

/**
 * Tests a fix for FLINK-1761.
 *
 * <p>FLINK-1761 discovered an IndexOutOfBoundsException, when receiving buffers of size 0.
 */
@Test
public void testReceiveEmptyBuffer() throws Exception {
	// Minimal mock of a remote input channel
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(TestBufferFactory.createBuffer(0));

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	// An empty buffer of size 0
	final Buffer emptyBuffer = TestBufferFactory.createBuffer(0);

	final int backlog = -1;
	final BufferResponse receivedBuffer = createBufferResponse(
		emptyBuffer, 0, inputChannel.getInputChannelId(), backlog);

	final PartitionRequestClientHandler client = new PartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	// Read the empty buffer
	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);

	// This should not throw an exception
	verify(inputChannel, never()).onError(any(Throwable.class));
	verify(inputChannel, times(1)).onEmptyBuffer(0, backlog);
}
 

/**
 * Tests a fix for FLINK-1761.
 *
 * <p>FLINK-1761 discovered an IndexOutOfBoundsException, when receiving buffers of size 0.
 */
@Test
public void testReceiveEmptyBuffer() throws Exception {
	// Minimal mock of a remote input channel
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(TestBufferFactory.createBuffer(0));

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	// An empty buffer of size 0
	final Buffer emptyBuffer = TestBufferFactory.createBuffer(0);

	final int backlog = 2;
	final BufferResponse receivedBuffer = createBufferResponse(
		emptyBuffer, 0, inputChannel.getInputChannelId(), backlog);

	final CreditBasedPartitionRequestClientHandler client = new CreditBasedPartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	// Read the empty buffer
	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);

	// This should not throw an exception
	verify(inputChannel, never()).onError(any(Throwable.class));
	verify(inputChannel, times(1)).onEmptyBuffer(0, backlog);
}
 

@Test
public void testExceptionOnReordering() throws Exception {
	// Setup
	final SingleInputGate inputGate = mock(SingleInputGate.class);
	final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);
	final Buffer buffer = TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE);

	// The test
	inputChannel.onBuffer(buffer.retainBuffer(), 0, -1);

	// This does not yet throw the exception, but sets the error at the channel.
	inputChannel.onBuffer(buffer, 29, -1);

	try {
		inputChannel.getNextBuffer();

		fail("Did not throw expected exception after enqueuing an out-of-order buffer.");
	}
	catch (Exception expected) {
		assertFalse(buffer.isRecycled());
		// free remaining buffer instances
		inputChannel.releaseAllResources();
		assertTrue(buffer.isRecycled());
	}

	// Need to notify the input gate for the out-of-order buffer as well. Otherwise the
	// receiving task will not notice the error.
	verify(inputGate, times(2)).notifyChannelNonEmpty(eq(inputChannel));
}
 

@Nullable
@Override
public BufferAndBacklog getNextBuffer() {
	int buffers = buffersInBacklog.decrementAndGet();
	return new BufferAndBacklog(
		TestBufferFactory.createBuffer(10),
		buffers > 0,
		buffers,
		false);
}
 

/**
 * Tests a fix for FLINK-1761.
 *
 * <p>FLINK-1761 discovered an IndexOutOfBoundsException, when receiving buffers of size 0.
 */
@Test
public void testReceiveEmptyBuffer() throws Exception {
	// Minimal mock of a remote input channel
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(TestBufferFactory.createBuffer(0));

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	// An empty buffer of size 0
	final Buffer emptyBuffer = TestBufferFactory.createBuffer(0);

	final int backlog = -1;
	final BufferResponse receivedBuffer = createBufferResponse(
		emptyBuffer, 0, inputChannel.getInputChannelId(), backlog);

	final PartitionRequestClientHandler client = new PartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	// Read the empty buffer
	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);

	// This should not throw an exception
	verify(inputChannel, never()).onError(any(Throwable.class));
	verify(inputChannel, times(1)).onEmptyBuffer(0, backlog);
}
 

/**
 * Tests a fix for FLINK-1761.
 *
 * <p>FLINK-1761 discovered an IndexOutOfBoundsException, when receiving buffers of size 0.
 */
@Test
public void testReceiveEmptyBuffer() throws Exception {
	// Minimal mock of a remote input channel
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(TestBufferFactory.createBuffer(0));

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	// An empty buffer of size 0
	final Buffer emptyBuffer = TestBufferFactory.createBuffer(0);

	final int backlog = 2;
	final BufferResponse receivedBuffer = createBufferResponse(
		emptyBuffer, 0, inputChannel.getInputChannelId(), backlog);

	final CreditBasedPartitionRequestClientHandler client = new CreditBasedPartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	// Read the empty buffer
	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);

	// This should not throw an exception
	verify(inputChannel, never()).onError(any(Throwable.class));
	verify(inputChannel, times(1)).onEmptyBuffer(0, backlog);
}
 

@Test
public void testExceptionOnReordering() throws Exception {
	// Setup
	final SingleInputGate inputGate = mock(SingleInputGate.class);
	final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);
	final Buffer buffer = TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE);

	// The test
	inputChannel.onBuffer(buffer.retainBuffer(), 0, -1);

	// This does not yet throw the exception, but sets the error at the channel.
	inputChannel.onBuffer(buffer, 29, -1);

	try {
		inputChannel.getNextBuffer();

		fail("Did not throw expected exception after enqueuing an out-of-order buffer.");
	}
	catch (Exception expected) {
		assertFalse(buffer.isRecycled());
		// free remaining buffer instances
		inputChannel.releaseAllResources();
		assertTrue(buffer.isRecycled());
	}

	// Need to notify the input gate for the out-of-order buffer as well. Otherwise the
	// receiving task will not notice the error.
	verify(inputGate, times(2)).notifyChannelNonEmpty(eq(inputChannel));
}
 

@Nullable
@Override
public BufferAndBacklog getNextBuffer() {
	int buffers = buffersInBacklog.decrementAndGet();
	return new BufferAndBacklog(
		TestBufferFactory.createBuffer(10),
		buffers > 0,
		buffers,
		false);
}
 

/**
 * Tests a fix for FLINK-1761.
 *
 * <p>FLINK-1761 discovered an IndexOutOfBoundsException, when receiving buffers of size 0.
 */
@Test
public void testReceiveEmptyBuffer() throws Exception {
	// Minimal mock of a remote input channel
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(TestBufferFactory.createBuffer(0));

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	// An empty buffer of size 0
	final Buffer emptyBuffer = TestBufferFactory.createBuffer(0);

	final CreditBasedPartitionRequestClientHandler client = new CreditBasedPartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	final int backlog = 2;
	final BufferResponse receivedBuffer = createBufferResponse(
		emptyBuffer,
		0,
		inputChannel.getInputChannelId(),
		backlog,
		new NetworkBufferAllocator(client));

	// Read the empty buffer
	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);

	// This should not throw an exception
	verify(inputChannel, never()).onError(any(Throwable.class));
	verify(inputChannel, times(1)).onEmptyBuffer(0, backlog);
}
 

/**
 * Repeatedly spawns two tasks: one to call <tt>function</tt> and the other to release the
 * channel concurrently. We do this repeatedly to provoke races.
 *
 * @param numberOfRepetitions how often to repeat the test
 * @param function function to call concurrently to {@link RemoteInputChannel#releaseAllResources()}
 */
private void testConcurrentReleaseAndSomething(
		final int numberOfRepetitions,
		TriFunction<RemoteInputChannel, Buffer, Integer, Object> function) throws Exception {

	// Setup
	final ExecutorService executor = Executors.newFixedThreadPool(2);
	final Buffer buffer = TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE);

	try {
		// Test
		final SingleInputGate inputGate = mock(SingleInputGate.class);

		for (int i = 0; i < numberOfRepetitions; i++) {
			final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);

			final Callable<Void> enqueueTask = () -> {
				while (true) {
					for (int j = 0; j < 128; j++) {
						// this is the same buffer over and over again which will be
						// recycled by the RemoteInputChannel
						Object obj = function.apply(inputChannel, buffer.retainBuffer(), j);
						if (obj instanceof NotificationResult && obj == NotificationResult.BUFFER_NOT_USED) {
							buffer.recycleBuffer();
						}
					}

					if (inputChannel.isReleased()) {
						return null;
					}
				}
			};

			final Callable<Void> releaseTask = () -> {
				inputChannel.releaseAllResources();
				return null;
			};

			// Submit tasks and wait to finish
			List<Future<Void>> results = Lists.newArrayListWithCapacity(2);

			results.add(executor.submit(enqueueTask));
			results.add(executor.submit(releaseTask));

			for (Future<Void> result : results) {
				result.get();
			}

			assertEquals("Resource leak during concurrent release and notifyBufferAvailable.",
				0, inputChannel.getNumberOfQueuedBuffers());
		}
	}
	finally {
		executor.shutdown();
		assertFalse(buffer.isRecycled());
		buffer.recycleBuffer();
		assertTrue(buffer.isRecycled());
	}
}
 

/**
 * Repeatedly spawns two tasks: one to call <tt>function</tt> and the other to release the
 * channel concurrently. We do this repeatedly to provoke races.
 *
 * @param numberOfRepetitions how often to repeat the test
 * @param function function to call concurrently to {@link RemoteInputChannel#releaseAllResources()}
 */
private void testConcurrentReleaseAndSomething(
		final int numberOfRepetitions,
		TriFunction<RemoteInputChannel, Buffer, Integer, Object> function) throws Exception {

	// Setup
	final ExecutorService executor = Executors.newFixedThreadPool(2);
	final Buffer buffer = TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE);

	try {
		// Test
		final SingleInputGate inputGate = mock(SingleInputGate.class);

		for (int i = 0; i < numberOfRepetitions; i++) {
			final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);

			final Callable<Void> enqueueTask = () -> {
				while (true) {
					for (int j = 0; j < 128; j++) {
						// this is the same buffer over and over again which will be
						// recycled by the RemoteInputChannel
						Object obj = function.apply(inputChannel, buffer.retainBuffer(), j);
						if (obj instanceof NotificationResult && obj == NotificationResult.BUFFER_NOT_USED) {
							buffer.recycleBuffer();
						}
					}

					if (inputChannel.isReleased()) {
						return null;
					}
				}
			};

			final Callable<Void> releaseTask = () -> {
				inputChannel.releaseAllResources();
				return null;
			};

			// Submit tasks and wait to finish
			List<Future<Void>> results = Lists.newArrayListWithCapacity(2);

			results.add(executor.submit(enqueueTask));
			results.add(executor.submit(releaseTask));

			for (Future<Void> result : results) {
				result.get();
			}

			assertEquals("Resource leak during concurrent release and notifyBufferAvailable.",
				0, inputChannel.getNumberOfQueuedBuffers());
		}
	}
	finally {
		executor.shutdown();
		assertFalse(buffer.isRecycled());
		buffer.recycleBuffer();
		assertTrue(buffer.isRecycled());
	}
}
 

/**
 * Repeatedly spawns two tasks: one to call <tt>function</tt> and the other to release the
 * channel concurrently. We do this repeatedly to provoke races.
 *
 * @param numberOfRepetitions how often to repeat the test
 * @param function function to call concurrently to {@link RemoteInputChannel#releaseAllResources()}
 */
private void testConcurrentReleaseAndSomething(
		final int numberOfRepetitions,
		TriFunction<RemoteInputChannel, Buffer, Integer, Object> function) throws Exception {

	// Setup
	final ExecutorService executor = Executors.newFixedThreadPool(2);
	final Buffer buffer = TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE);

	try {
		// Test
		final SingleInputGate inputGate = createSingleInputGate(1);

		for (int i = 0; i < numberOfRepetitions; i++) {
			final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);

			final Callable<Void> enqueueTask = () -> {
				while (true) {
					for (int j = 0; j < 128; j++) {
						// this is the same buffer over and over again which will be
						// recycled by the RemoteInputChannel
						Object obj = function.apply(inputChannel, buffer.retainBuffer(), j);
						if (obj instanceof NotificationResult && obj == NotificationResult.BUFFER_NOT_USED) {
							buffer.recycleBuffer();
						}
					}

					if (inputChannel.isReleased()) {
						return null;
					}
				}
			};

			final Callable<Void> releaseTask = () -> {
				inputChannel.releaseAllResources();
				return null;
			};

			// Submit tasks and wait to finish
			List<Future<Void>> results = Lists.newArrayListWithCapacity(2);

			results.add(executor.submit(enqueueTask));
			results.add(executor.submit(releaseTask));

			for (Future<Void> result : results) {
				result.get();
			}

			assertEquals("Resource leak during concurrent release and notifyBufferAvailable.",
				0, inputChannel.getNumberOfQueuedBuffers());
		}
	}
	finally {
		executor.shutdown();
		assertFalse(buffer.isRecycled());
		buffer.recycleBuffer();
		assertTrue(buffer.isRecycled());
	}
}
 

private BufferOrEvent createBuffer(int channel) {
	final int size = sizeCounter++;
	return new BufferOrEvent(TestBufferFactory.createBuffer(size), new InputChannelInfo(0, channel));
}