Java Code Examples for org.apache.flink.runtime.io.disk.iomanager.IOManager#getSpillingDirectories()

/**
 * Creates a new buffer spiller, spilling to one of the I/O manager's temp directories.
 *
 * @param ioManager The I/O manager for access to the temp directories.
 * @param pageSize The page size used to re-create spilled buffers.
 * @throws IOException Thrown if the temp files for spilling cannot be initialized.
 */
public BufferSpiller(IOManager ioManager, int pageSize) throws IOException {
	this.pageSize = pageSize;

	this.readBuffer = ByteBuffer.allocateDirect(READ_BUFFER_SIZE);
	this.readBuffer.order(ByteOrder.LITTLE_ENDIAN);

	this.headBuffer = ByteBuffer.allocateDirect(16);
	this.headBuffer.order(ByteOrder.LITTLE_ENDIAN);

	File[] tempDirs = ioManager.getSpillingDirectories();
	this.tempDir = tempDirs[DIRECTORY_INDEX.getAndIncrement() % tempDirs.length];

	byte[] rndBytes = new byte[32];
	ThreadLocalRandom.current().nextBytes(rndBytes);
	this.spillFilePrefix = StringUtils.byteToHexString(rndBytes) + '.';

	// prepare for first contents
	createSpillingChannel();
}
 

/**
 * Creates a new {@link BufferSpiller}, spilling to one of the I/O manager's temp directories.
 *
 * @param ioManager The I/O manager for access to the temp directories.
 * @param pageSize The page size used to re-create spilled buffers.
 * @param maxBufferedBytes The maximum bytes to be buffered before the checkpoint aborts.
 * @param taskName The task name for logging.
 * @throws IOException Thrown if the temp files for spilling cannot be initialized.
 */
public BufferSpiller(IOManager ioManager, int pageSize, long maxBufferedBytes, String taskName) throws IOException {
	super(maxBufferedBytes, taskName);
	this.pageSize = pageSize;

	this.readBuffer = ByteBuffer.allocateDirect(READ_BUFFER_SIZE);
	this.readBuffer.order(ByteOrder.LITTLE_ENDIAN);

	this.headBuffer = ByteBuffer.allocateDirect(16);
	this.headBuffer.order(ByteOrder.LITTLE_ENDIAN);

	File[] tempDirs = ioManager.getSpillingDirectories();
	this.tempDir = tempDirs[DIRECTORY_INDEX.getAndIncrement() % tempDirs.length];

	byte[] rndBytes = new byte[32];
	ThreadLocalRandom.current().nextBytes(rndBytes);
	this.spillFilePrefix = StringUtils.byteToHexString(rndBytes) + '.';

	// prepare for first contents
	createSpillingChannel();
}
 

private static void checkNoTempFilesRemain(IOManager ioManager) {
	for (File dir : ioManager.getSpillingDirectories()) {
		for (String file : dir.list()) {
			if (file != null && !(file.equals(".") || file.equals(".."))) {
				fail("hash table did not clean up temp files. remaining file: " + file);
			}
		}
	}
}
 

private static void checkNoTempFilesRemain(IOManager ioManager) {
	for (File dir : ioManager.getSpillingDirectories()) {
		for (String file : dir.list()) {
			if (file != null && !(file.equals(".") || file.equals(".."))) {
				fail("hash table did not clean up temp files. remaining file: " + file);
			}
		}
	}
}
 

private static void checkNoTempFilesRemain(IOManager ioManager) {
	for (File dir : ioManager.getSpillingDirectories()) {
		for (String file : dir.list()) {
			if (file != null && !(file.equals(".") || file.equals(".."))) {
				fail("hash table did not clean up temp files. remaining file: " + file);
			}
		}
	}
}
 

private static void checkNoTempFilesRemain(IOManager ioManager) {
	for (File dir : ioManager.getSpillingDirectories()) {
		for (String file : dir.list()) {
			if (file != null && !(file.equals(".") || file.equals(".."))) {
				fail("hash table did not clean up temp files. remaining file: " + file);
			}
		}
	}
}
 

private static void checkNoTempFilesRemain(IOManager ioManager) {
	for (File dir : ioManager.getSpillingDirectories()) {
		for (String file : dir.list()) {
			if (file != null && !(file.equals(".") || file.equals(".."))) {
				fail("hash table did not clean up temp files. remaining file: " + file);
			}
		}
	}
}
 

@Test
public void testShouldShutDownTaskManagerServicesInPostStop() throws Exception {
	final TaskSlotTableImpl<Task> taskSlotTable = TaskSlotUtils.createTaskSlotTable(1);

	final JobLeaderService jobLeaderService = new DefaultJobLeaderService(unresolvedTaskManagerLocation, RetryingRegistrationConfiguration.defaultConfiguration());

	final IOManager ioManager = new IOManagerAsync(tmp.newFolder().getAbsolutePath());

	final TaskExecutorLocalStateStoresManager localStateStoresManager = new TaskExecutorLocalStateStoresManager(
		false,
		ioManager.getSpillingDirectories(),
		Executors.directExecutor());

	nettyShuffleEnvironment.start();

	final KvStateService kvStateService = new KvStateService(new KvStateRegistry(), null, null);
	kvStateService.start();

	final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder()
		.setUnresolvedTaskManagerLocation(unresolvedTaskManagerLocation)
		.setIoManager(ioManager)
		.setShuffleEnvironment(nettyShuffleEnvironment)
		.setKvStateService(kvStateService)
		.setTaskSlotTable(taskSlotTable)
		.setJobLeaderService(jobLeaderService)
		.setTaskStateManager(localStateStoresManager)
		.build();

	final TaskExecutor taskManager = createTaskExecutor(taskManagerServices);

	try {
		taskManager.start();
	} finally {
		RpcUtils.terminateRpcEndpoint(taskManager, timeout);
	}

	assertThat(taskSlotTable.isClosed(), is(true));
	assertThat(nettyShuffleEnvironment.isClosed(), is(true));
	assertThat(kvStateService.isShutdown(), is(true));
}
 

private static void checkNoTempFilesRemain(IOManager ioManager) {
	for (File dir : ioManager.getSpillingDirectories()) {
		for (String file : dir.list()) {
			if (file != null && !(file.equals(".") || file.equals(".."))) {
				fail("hash table did not clean up temp files. remaining file: " + file);
			}
		}
	}
}
 

@Test
public void testShouldShutDownTaskManagerServicesInPostStop() throws Exception {
	final TaskSlotTable taskSlotTable = new TaskSlotTable(Collections.singleton(ResourceProfile.UNKNOWN), timerService);

	final JobLeaderService jobLeaderService = new JobLeaderService(taskManagerLocation, RetryingRegistrationConfiguration.defaultConfiguration());

	final IOManager ioManager = new IOManagerAsync(tmp.newFolder().getAbsolutePath());

	final TaskExecutorLocalStateStoresManager localStateStoresManager = new TaskExecutorLocalStateStoresManager(
		false,
		ioManager.getSpillingDirectories(),
		Executors.directExecutor());

	final MemoryManager memoryManager = new MemoryManager(
		4096,
		1,
		4096,
		MemoryType.HEAP,
		false);

	final NetworkEnvironment networkEnvironment = new NetworkEnvironment(
		1,
		1,
		0,
		0,
		2,
		8,
		true);
	networkEnvironment.start();

	final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder()
		.setTaskManagerLocation(taskManagerLocation)
		.setMemoryManager(memoryManager)
		.setIoManager(ioManager)
		.setNetworkEnvironment(networkEnvironment)
		.setTaskSlotTable(taskSlotTable)
		.setJobLeaderService(jobLeaderService)
		.setTaskStateManager(localStateStoresManager)
		.build();

	final long heartbeatInterval = 1000L;
	final long heartbeatTimeout = 1000L;
	final HeartbeatServices heartbeatServices = new HeartbeatServices(heartbeatInterval, heartbeatTimeout);

	final TaskExecutor taskManager = new TaskExecutor(
		rpc,
		taskManagerConfiguration,
		haServices,
		taskManagerServices,
		heartbeatServices,
		UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(),
		null,
		dummyBlobCacheService,
		testingFatalErrorHandler);

	try {
		taskManager.start();
	} finally {
		RpcUtils.terminateRpcEndpoint(taskManager, timeout);
	}

	assertThat(memoryManager.isShutdown(), is(true));
	assertThat(networkEnvironment.isShutdown(), is(true));
	assertThat(ioManager.isProperlyShutDown(), is(true));
}
 

@Test
public void testShouldShutDownTaskManagerServicesInPostStop() throws Exception {
	final TaskSlotTable taskSlotTable = new TaskSlotTable(Collections.singleton(ResourceProfile.UNKNOWN), timerService);

	final JobLeaderService jobLeaderService = new JobLeaderService(taskManagerLocation, RetryingRegistrationConfiguration.defaultConfiguration());

	final IOManager ioManager = new IOManagerAsync(tmp.newFolder().getAbsolutePath());

	final TaskExecutorLocalStateStoresManager localStateStoresManager = new TaskExecutorLocalStateStoresManager(
		false,
		ioManager.getSpillingDirectories(),
		Executors.directExecutor());

	final MemoryManager memoryManager = new MemoryManager(
		4096,
		1,
		4096,
		MemoryType.HEAP,
		false);

	nettyShuffleEnvironment.start();

	final KvStateService kvStateService = new KvStateService(new KvStateRegistry(), null, null);
	kvStateService.start();

	final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder()
		.setTaskManagerLocation(taskManagerLocation)
		.setMemoryManager(memoryManager)
		.setIoManager(ioManager)
		.setShuffleEnvironment(nettyShuffleEnvironment)
		.setKvStateService(kvStateService)
		.setTaskSlotTable(taskSlotTable)
		.setJobLeaderService(jobLeaderService)
		.setTaskStateManager(localStateStoresManager)
		.build();

	final TaskExecutor taskManager = createTaskExecutor(taskManagerServices);

	try {
		taskManager.start();
	} finally {
		RpcUtils.terminateRpcEndpoint(taskManager, timeout);
	}

	assertThat(memoryManager.isShutdown(), is(true));
	assertThat(nettyShuffleEnvironment.isClosed(), is(true));
	assertThat(kvStateService.isShutdown(), is(true));
}