Java Code Examples for org.apache.flink.shaded.guava18.com.google.common.collect.Lists#newArrayListWithCapacity()

public TestPartitionProducerBufferSource(
		int parallelism,
		BufferProvider bufferProvider,
		int numberOfBuffersToProduce) {

	this.bufferProvider = bufferProvider;
	this.channelIndexes = Lists.newArrayListWithCapacity(
			parallelism * numberOfBuffersToProduce);

	// Array of channel indexes to produce buffers for
	for (byte i = 0; i < parallelism; i++) {
		for (int j = 0; j < numberOfBuffersToProduce; j++) {
			channelIndexes.add(i);
		}
	}

	// Random buffer to channel ordering
	Collections.shuffle(channelIndexes);
}
 

public TestPartitionProducerBufferSource(
		int parallelism,
		BufferProvider bufferProvider,
		int numberOfBuffersToProduce) {

	this.bufferProvider = bufferProvider;
	this.channelIndexes = Lists.newArrayListWithCapacity(
			parallelism * numberOfBuffersToProduce);

	// Array of channel indexes to produce buffers for
	for (byte i = 0; i < parallelism; i++) {
		for (int j = 0; j < numberOfBuffersToProduce; j++) {
			channelIndexes.add(i);
		}
	}

	// Random buffer to channel ordering
	Collections.shuffle(channelIndexes);
}
 

public TestPartitionProducerBufferSource(
		int parallelism,
		BufferProvider bufferProvider,
		int numberOfBuffersToProduce) {

	this.bufferProvider = bufferProvider;
	this.channelIndexes = Lists.newArrayListWithCapacity(
			parallelism * numberOfBuffersToProduce);

	// Array of channel indexes to produce buffers for
	for (byte i = 0; i < parallelism; i++) {
		for (int j = 0; j < numberOfBuffersToProduce; j++) {
			channelIndexes.add(i);
		}
	}

	// Random buffer to channel ordering
	Collections.shuffle(channelIndexes);
}
 

@Override
public void invoke() throws Exception {
	List<RecordWriter<IntValue>> writers = Lists.newArrayListWithCapacity(2);

	// The order of intermediate result creation in the job graph specifies which produced
	// result partition is pipelined/blocking.
	final RecordWriter<IntValue> pipelinedWriter =
			new RecordWriter<>(getEnvironment().getWriter(0));

	final RecordWriter<IntValue> blockingWriter =
			new RecordWriter<>(getEnvironment().getWriter(1));

	writers.add(pipelinedWriter);
	writers.add(blockingWriter);

	final int numberOfTimesToSend = getTaskConfiguration().getInteger(CONFIG_KEY, 0);

	final IntValue subtaskIndex = new IntValue(
			getEnvironment().getTaskInfo().getIndexOfThisSubtask());

	// Produce the first intermediate result and then the second in a serial fashion.
	for (RecordWriter<IntValue> writer : writers) {
		try {
			for (int i = 0; i < numberOfTimesToSend; i++) {
				writer.emit(subtaskIndex);
			}
			writer.flushAll();
		}
		finally {
			writer.clearBuffers();
		}
	}
}
 

/**
 * Submits all the callable tasks to the executor and waits for the results.
 *
 * @param executor The executor service for running tasks.
 * @param tasks The callable tasks to be submitted and executed.
 */
private void submitTasksAndWaitForResults(ExecutorService executor, Callable[] tasks) throws Exception {
	final List<Future> results = Lists.newArrayListWithCapacity(tasks.length);

	for (Callable task : tasks) {
		//noinspection unchecked
		results.add(executor.submit(task));
	}

	for (Future result : results) {
		result.get();
	}
}
 

@Override
public void invoke() throws Exception {
	List<RecordWriter<IntValue>> writers = Lists.newArrayListWithCapacity(2);

	// The order of intermediate result creation in the job graph specifies which produced
	// result partition is pipelined/blocking.
	final RecordWriter<IntValue> pipelinedWriter = new RecordWriterBuilder().build(getEnvironment().getWriter(0));
	final RecordWriter<IntValue> blockingWriter = new RecordWriterBuilder().build(getEnvironment().getWriter(1));
	writers.add(pipelinedWriter);
	writers.add(blockingWriter);

	final int numberOfTimesToSend = getTaskConfiguration().getInteger(CONFIG_KEY, 0);

	final IntValue subtaskIndex = new IntValue(
			getEnvironment().getTaskInfo().getIndexOfThisSubtask());

	// Produce the first intermediate result and then the second in a serial fashion.
	for (RecordWriter<IntValue> writer : writers) {
		try {
			for (int i = 0; i < numberOfTimesToSend; i++) {
				writer.emit(subtaskIndex);
			}
			writer.flushAll();
		}
		finally {
			writer.clearBuffers();
		}
	}
}
 

/**
 * Submits all the callable tasks to the executor and waits for the results.
 *
 * @param executor The executor service for running tasks.
 * @param tasks The callable tasks to be submitted and executed.
 */
private void submitTasksAndWaitForResults(ExecutorService executor, Callable[] tasks) throws Exception {
	final List<Future> results = Lists.newArrayListWithCapacity(tasks.length);

	for (Callable task : tasks) {
		//noinspection unchecked
		results.add(executor.submit(task));
	}

	for (Future result : results) {
		result.get();
	}
}
 

@Override
public void invoke() throws Exception {
	List<RecordWriter<IntValue>> writers = Lists.newArrayListWithCapacity(2);

	// The order of intermediate result creation in the job graph specifies which produced
	// result partition is pipelined/blocking.
	final RecordWriter<IntValue> pipelinedWriter = new RecordWriterBuilder<IntValue>().build(getEnvironment().getWriter(0));
	final RecordWriter<IntValue> blockingWriter = new RecordWriterBuilder<IntValue>().build(getEnvironment().getWriter(1));
	writers.add(pipelinedWriter);
	writers.add(blockingWriter);

	final int numberOfTimesToSend = getTaskConfiguration().getInteger(CONFIG_KEY, 0);

	final IntValue subtaskIndex = new IntValue(
			getEnvironment().getTaskInfo().getIndexOfThisSubtask());

	// Produce the first intermediate result and then the second in a serial fashion.
	for (RecordWriter<IntValue> writer : writers) {
		try {
			for (int i = 0; i < numberOfTimesToSend; i++) {
				writer.emit(subtaskIndex);
			}
			writer.flushAll();
		}
		finally {
			writer.clearBuffers();
		}
	}
}
 

/**
 * Submits all the callable tasks to the executor and waits for the results.
 *
 * @param executor The executor service for running tasks.
 * @param tasks The callable tasks to be submitted and executed.
 */
static void submitTasksAndWaitForResults(ExecutorService executor, Callable[] tasks) throws Exception {
	final List<Future> results = Lists.newArrayListWithCapacity(tasks.length);

	for (Callable task : tasks) {
		//noinspection unchecked
		results.add(executor.submit(task));
	}

	for (Future result : results) {
		result.get();
	}
}
 

/**
 * Repeatedly spawns two tasks: one to call <tt>function</tt> and the other to release the
 * channel concurrently. We do this repeatedly to provoke races.
 *
 * @param numberOfRepetitions how often to repeat the test
 * @param function function to call concurrently to {@link RemoteInputChannel#releaseAllResources()}
 */
private void testConcurrentReleaseAndSomething(
		final int numberOfRepetitions,
		TriFunction<RemoteInputChannel, Buffer, Integer, Object> function) throws Exception {

	// Setup
	final ExecutorService executor = Executors.newFixedThreadPool(2);
	final Buffer buffer = TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE);

	try {
		// Test
		final SingleInputGate inputGate = mock(SingleInputGate.class);

		for (int i = 0; i < numberOfRepetitions; i++) {
			final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);

			final Callable<Void> enqueueTask = () -> {
				while (true) {
					for (int j = 0; j < 128; j++) {
						// this is the same buffer over and over again which will be
						// recycled by the RemoteInputChannel
						Object obj = function.apply(inputChannel, buffer.retainBuffer(), j);
						if (obj instanceof NotificationResult && obj == NotificationResult.BUFFER_NOT_USED) {
							buffer.recycleBuffer();
						}
					}

					if (inputChannel.isReleased()) {
						return null;
					}
				}
			};

			final Callable<Void> releaseTask = () -> {
				inputChannel.releaseAllResources();
				return null;
			};

			// Submit tasks and wait to finish
			List<Future<Void>> results = Lists.newArrayListWithCapacity(2);

			results.add(executor.submit(enqueueTask));
			results.add(executor.submit(releaseTask));

			for (Future<Void> result : results) {
				result.get();
			}

			assertEquals("Resource leak during concurrent release and notifyBufferAvailable.",
				0, inputChannel.getNumberOfQueuedBuffers());
		}
	}
	finally {
		executor.shutdown();
		assertFalse(buffer.isRecycled());
		buffer.recycleBuffer();
		assertTrue(buffer.isRecycled());
	}
}
 

/**
 * Repeatedly spawns two tasks: one to call <tt>function</tt> and the other to release the
 * channel concurrently. We do this repeatedly to provoke races.
 *
 * @param numberOfRepetitions how often to repeat the test
 * @param function function to call concurrently to {@link RemoteInputChannel#releaseAllResources()}
 */
private void testConcurrentReleaseAndSomething(
		final int numberOfRepetitions,
		TriFunction<RemoteInputChannel, Buffer, Integer, Object> function) throws Exception {

	// Setup
	final ExecutorService executor = Executors.newFixedThreadPool(2);
	final Buffer buffer = TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE);

	try {
		// Test
		final SingleInputGate inputGate = mock(SingleInputGate.class);

		for (int i = 0; i < numberOfRepetitions; i++) {
			final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);

			final Callable<Void> enqueueTask = () -> {
				while (true) {
					for (int j = 0; j < 128; j++) {
						// this is the same buffer over and over again which will be
						// recycled by the RemoteInputChannel
						Object obj = function.apply(inputChannel, buffer.retainBuffer(), j);
						if (obj instanceof NotificationResult && obj == NotificationResult.BUFFER_NOT_USED) {
							buffer.recycleBuffer();
						}
					}

					if (inputChannel.isReleased()) {
						return null;
					}
				}
			};

			final Callable<Void> releaseTask = () -> {
				inputChannel.releaseAllResources();
				return null;
			};

			// Submit tasks and wait to finish
			List<Future<Void>> results = Lists.newArrayListWithCapacity(2);

			results.add(executor.submit(enqueueTask));
			results.add(executor.submit(releaseTask));

			for (Future<Void> result : results) {
				result.get();
			}

			assertEquals("Resource leak during concurrent release and notifyBufferAvailable.",
				0, inputChannel.getNumberOfQueuedBuffers());
		}
	}
	finally {
		executor.shutdown();
		assertFalse(buffer.isRecycled());
		buffer.recycleBuffer();
		assertTrue(buffer.isRecycled());
	}
}
 

/**
 * Repeatedly spawns two tasks: one to call <tt>function</tt> and the other to release the
 * channel concurrently. We do this repeatedly to provoke races.
 *
 * @param numberOfRepetitions how often to repeat the test
 * @param function function to call concurrently to {@link RemoteInputChannel#releaseAllResources()}
 */
private void testConcurrentReleaseAndSomething(
		final int numberOfRepetitions,
		TriFunction<RemoteInputChannel, Buffer, Integer, Object> function) throws Exception {

	// Setup
	final ExecutorService executor = Executors.newFixedThreadPool(2);
	final Buffer buffer = TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE);

	try {
		// Test
		final SingleInputGate inputGate = createSingleInputGate(1);

		for (int i = 0; i < numberOfRepetitions; i++) {
			final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);

			final Callable<Void> enqueueTask = () -> {
				while (true) {
					for (int j = 0; j < 128; j++) {
						// this is the same buffer over and over again which will be
						// recycled by the RemoteInputChannel
						Object obj = function.apply(inputChannel, buffer.retainBuffer(), j);
						if (obj instanceof NotificationResult && obj == NotificationResult.BUFFER_NOT_USED) {
							buffer.recycleBuffer();
						}
					}

					if (inputChannel.isReleased()) {
						return null;
					}
				}
			};

			final Callable<Void> releaseTask = () -> {
				inputChannel.releaseAllResources();
				return null;
			};

			// Submit tasks and wait to finish
			List<Future<Void>> results = Lists.newArrayListWithCapacity(2);

			results.add(executor.submit(enqueueTask));
			results.add(executor.submit(releaseTask));

			for (Future<Void> result : results) {
				result.get();
			}

			assertEquals("Resource leak during concurrent release and notifyBufferAvailable.",
				0, inputChannel.getNumberOfQueuedBuffers());
		}
	}
	finally {
		executor.shutdown();
		assertFalse(buffer.isRecycled());
		buffer.recycleBuffer();
		assertTrue(buffer.isRecycled());
	}
}
 

/**
 * Tests the consumption of multiple subpartitions via local input channels.
 *
 * <p>Multiple producer tasks produce pipelined partitions, which are consumed by multiple
 * tasks via local input channels.
 */
@Test
public void testConcurrentConsumeMultiplePartitions() throws Exception {
	// Config
	final int parallelism = 32;
	final int producerBufferPoolSize = parallelism + 1;
	final int numberOfBuffersPerChannel = 1024;

	checkArgument(parallelism >= 1);
	checkArgument(producerBufferPoolSize >= parallelism);
	checkArgument(numberOfBuffersPerChannel >= 1);

	// Setup
	// One thread per produced partition and one per consumer
	final ExecutorService executor = Executors.newFixedThreadPool(2 * parallelism);

	final NetworkBufferPool networkBuffers = new NetworkBufferPool(
		(parallelism * producerBufferPoolSize) + (parallelism * parallelism),
		TestBufferFactory.BUFFER_SIZE, 1);

	final ResultPartitionManager partitionManager = new ResultPartitionManager();

	final ResultPartitionID[] partitionIds = new ResultPartitionID[parallelism];
	final TestPartitionProducer[] partitionProducers = new TestPartitionProducer[parallelism];

	// Create all partitions
	for (int i = 0; i < parallelism; i++) {
		partitionIds[i] = new ResultPartitionID();

		final ResultPartition partition = new ResultPartitionBuilder()
			.setResultPartitionId(partitionIds[i])
			.setNumberOfSubpartitions(parallelism)
			.setNumTargetKeyGroups(parallelism)
			.setResultPartitionManager(partitionManager)
			.setBufferPoolFactory(p ->
				networkBuffers.createBufferPool(producerBufferPoolSize, producerBufferPoolSize))
			.build();

		// Create a buffer pool for this partition
		partition.setup();

		// Create the producer
		partitionProducers[i] = new TestPartitionProducer(
			partition,
			false,
			new TestPartitionProducerBufferSource(
				parallelism,
				partition.getBufferPool(),
				numberOfBuffersPerChannel)
		);
	}

	// Test
	try {
		// Submit producer tasks
		List<CompletableFuture<?>> results = Lists.newArrayListWithCapacity(
			parallelism + 1);

		for (int i = 0; i < parallelism; i++) {
			results.add(CompletableFuture.supplyAsync(
				CheckedSupplier.unchecked(partitionProducers[i]::call), executor));
		}

		// Submit consumer
		for (int i = 0; i < parallelism; i++) {
			final TestLocalInputChannelConsumer consumer = new TestLocalInputChannelConsumer(
				i,
				parallelism,
				numberOfBuffersPerChannel,
				networkBuffers.createBufferPool(parallelism, parallelism),
				partitionManager,
				new TaskEventDispatcher(),
				partitionIds);

			results.add(CompletableFuture.supplyAsync(CheckedSupplier.unchecked(consumer::call), executor));
		}

		FutureUtils.waitForAll(results).get();
	}
	finally {
		networkBuffers.destroyAllBufferPools();
		networkBuffers.destroy();
		executor.shutdown();
	}
}