org.apache.flink.runtime.io.disk.iomanager.IOManager Java Examples

/**
 * Creates the spilling thread.
 * 
 * @param exceptionHandler The exception handler to call for all exceptions.
 * @param queues The queues used to pass buffers between the threads.
 * @param parentTask The task that started this thread. If non-null, it is used to register this thread.
 * @param memManager The memory manager used to allocate buffers for the readers and writers.
 * @param ioManager The I/I manager used to instantiate readers and writers from.
 * @param serializer
 * @param comparator
 * @param sortReadMemory
 * @param writeMemory
 * @param maxNumFileHandles
 */
public SpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
		AbstractInvokable parentTask, MemoryManager memManager, IOManager ioManager, 
		TypeSerializer<E> serializer, TypeComparator<E> comparator, 
		List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxNumFileHandles)
{
	super(exceptionHandler, "SortMerger spilling thread", queues, parentTask);
	this.memManager = memManager;
	this.ioManager = ioManager;
	this.serializer = serializer;
	this.comparator = comparator;
	this.mergeReadMemory = sortReadMemory;
	this.writeMemory = writeMemory;
	this.maxFanIn = maxNumFileHandles;
	this.numWriteBuffersToCluster = writeMemory.size() >= 4 ? writeMemory.size() / 2 : 1;
}
 

public LongHybridHashTable(
		Configuration conf,
		Object owner,
		BinaryRowDataSerializer buildSideSerializer,
		BinaryRowDataSerializer probeSideSerializer,
		MemoryManager memManager,
		long reservedMemorySize,
		IOManager ioManager,
		int avgRecordLen,
		long buildRowCount) {
	super(conf, owner, memManager, reservedMemorySize, ioManager, avgRecordLen, buildRowCount, false);
	this.buildSideSerializer = buildSideSerializer;
	this.probeSideSerializer = probeSideSerializer;

	this.partitionsBeingBuilt = new ArrayList<>();
	this.partitionsPending = new ArrayList<>();

	createPartitions(initPartitionFanOut, 0);
}
 

public NonReusingBuildSecondReOpenableHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V1, V2> pairComparator,
		MemoryManager memManager,
		IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	super(firstInput, secondInput, serializer1, comparator1, serializer2,
			comparator2, pairComparator, memManager, ioManager, ownerTask,
			memoryFraction, probeSideOuterJoin, buildSideOuterJoin, useBitmapFilters);
	
	reopenHashTable = (ReOpenableMutableHashTable<V2, V1>) hashJoin;
}
 

public NonReusingBuildSecondReOpenableHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V1, V2> pairComparator,
		MemoryManager memManager,
		IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	super(firstInput, secondInput, serializer1, comparator1, serializer2,
			comparator2, pairComparator, memManager, ioManager, ownerTask,
			memoryFraction, probeSideOuterJoin, buildSideOuterJoin, useBitmapFilters);
	
	reopenHashTable = (ReOpenableMutableHashTable<V2, V1>) hashJoin;
}
 

public ReusingMergeOuterJoinIterator(
		OuterJoinType outerJoinType,
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(outerJoinType, input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);

	this.copy1 = serializer1.createInstance();
	this.spillHeadCopy = serializer1.createInstance();
	this.copy2 = serializer2.createInstance();
	this.blockHeadCopy = serializer2.createInstance();
}
 

public AbstractBinaryExternalMerger(
		IOManager ioManager,
		int pageSize,
		int maxFanIn,
		SpillChannelManager channelManager,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize) {
	this.ioManager = ioManager;
	this.pageSize = pageSize;
	this.maxFanIn = maxFanIn;
	this.channelManager = channelManager;
	this.compressionEnable = compressionEnable;
	this.compressionCodecFactory = compressionCodecFactory;
	this.compressionBlockSize = compressionBlockSize;
}
 

/**
 * Creates the spilling thread.
 * 
 * @param exceptionHandler The exception handler to call for all exceptions.
 * @param queues The queues used to pass buffers between the threads.
 * @param parentTask The task that started this thread. If non-null, it is used to register this thread.
 * @param memManager The memory manager used to allocate buffers for the readers and writers.
 * @param ioManager The I/I manager used to instantiate readers and writers from.
 * @param serializer
 * @param comparator
 * @param sortReadMemory
 * @param writeMemory
 * @param maxNumFileHandles
 */
public SpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
		AbstractInvokable parentTask, MemoryManager memManager, IOManager ioManager, 
		TypeSerializer<E> serializer, TypeComparator<E> comparator, 
		List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxNumFileHandles)
{
	super(exceptionHandler, "SortMerger spilling thread", queues, parentTask);
	this.memManager = memManager;
	this.ioManager = ioManager;
	this.serializer = serializer;
	this.comparator = comparator;
	this.mergeReadMemory = sortReadMemory;
	this.writeMemory = writeMemory;
	this.maxFanIn = maxNumFileHandles;
	this.numWriteBuffersToCluster = writeMemory.size() >= 4 ? writeMemory.size() / 2 : 1;
}
 

protected UnilateralSortMerger(MemoryManager memoryManager, List<MemorySegment> memory,
		IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask, 
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		int numSortBuffers, int maxNumFileHandles,
		float startSpillingFraction, boolean noSpillingMemory, boolean handleLargeRecords,
		boolean objectReuseEnabled) throws IOException {
	this (
		memoryManager,
		memory,
		ioManager,
		input,
		parentTask,
		serializerFactory,
		comparator,
		numSortBuffers,
		maxNumFileHandles,
		startSpillingFraction,
		noSpillingMemory,
		handleLargeRecords,
		objectReuseEnabled,
		new DefaultInMemorySorterFactory<>(serializerFactory, comparator, THRESHOLD_FOR_IN_PLACE_SORTING));
}
 

/**
 * Spills this partition to disk. This method is invoked once after the initial open() method
 * 
 * @return Number of memorySegments in the writeBehindBuffers!
 */
int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException {
	this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap
	this.initialBuildSideChannel = targetChannel;
	
	initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers);
	
	final int numSegments = this.partitionBuffers.length;
	for (int i = 0; i < numSegments; i++) {
		initialBuildSideWriter.writeBlock(partitionBuffers[i]);
	}
	this.partitionBuffers = null;
	initialBuildSideWriter.close();
	// num partitions are now in the writeBehindBuffers. We propagate this information back
	return numSegments;
	
}
 

public AbstractBinaryExternalMerger(
		IOManager ioManager,
		int pageSize,
		int maxFanIn,
		SpillChannelManager channelManager,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize) {
	this.ioManager = ioManager;
	this.pageSize = pageSize;
	this.maxFanIn = maxFanIn;
	this.channelManager = channelManager;
	this.compressionEnable = compressionEnable;
	this.compressionCodecFactory = compressionCodecFactory;
	this.compressionBlockSize = compressionBlockSize;
}
 

public ReusingMergeOuterJoinIterator(
		OuterJoinType outerJoinType,
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(outerJoinType, input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);

	this.copy1 = serializer1.createInstance();
	this.spillHeadCopy = serializer1.createInstance();
	this.copy2 = serializer2.createInstance();
	this.blockHeadCopy = serializer2.createInstance();
}
 

public ReusingMergeInnerJoinIterator(
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);

	this.copy1 = serializer1.createInstance();
	this.spillHeadCopy = serializer1.createInstance();
	this.copy2 = serializer2.createInstance();
	this.blockHeadCopy = serializer2.createInstance();
}
 

public static AbstractChannelWriterOutputView createOutputView(
		IOManager ioManager,
		FileIOChannel.ID channel,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		BufferFileWriter bufferWriter = ioManager.createBufferFileWriter(channel);
		return new CompressedHeaderlessChannelWriterOutputView(
				bufferWriter,
				compressionCodecFactory,
				compressionBlockSize);
	} else {
		BlockChannelWriter<MemorySegment> blockWriter =
				ioManager.createBlockChannelWriter(channel);
		return new HeaderlessChannelWriterOutputView(
				blockWriter,
				Arrays.asList(
						allocateUnpooledSegment(segmentSize),
						allocateUnpooledSegment(segmentSize)
				),
				segmentSize
		);
	}
}
 

public CompressedBlockChannelReader(
		IOManager ioManager,
		ID channel,
		LinkedBlockingQueue<MemorySegment> blockQueue,
		BlockCompressionFactory codecFactory,
		int preferBlockSize,
		int segmentSize) throws IOException {
	this.reader = ioManager.createBufferFileReader(channel, this);
	this.blockQueue = blockQueue;
	copyCompress = preferBlockSize > segmentSize * 2;
	int blockSize = copyCompress ? preferBlockSize : segmentSize;
	this.decompressor = codecFactory.getDecompressor();
	cause = new AtomicReference<>();

	if (copyCompress) {
		this.buf = new byte[blockSize];
		this.bufWrapper = ByteBuffer.wrap(buf);
	}

	BlockCompressor compressor = codecFactory.getCompressor();
	for (int i = 0; i < 2; i++) {
		MemorySegment segment = MemorySegmentFactory.wrap(new byte[compressor.getMaxCompressedSize(blockSize)]);
		reader.readInto(new NetworkBuffer(segment, this));
	}
}
 

protected UnilateralSortMerger(MemoryManager memoryManager, List<MemorySegment> memory,
		IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask, 
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		int numSortBuffers, int maxNumFileHandles,
		float startSpillingFraction, boolean noSpillingMemory, boolean handleLargeRecords,
		boolean objectReuseEnabled) throws IOException {
	this (
		memoryManager,
		memory,
		ioManager,
		input,
		parentTask,
		serializerFactory,
		comparator,
		numSortBuffers,
		maxNumFileHandles,
		startSpillingFraction,
		noSpillingMemory,
		handleLargeRecords,
		objectReuseEnabled,
		new DefaultInMemorySorterFactory<>(serializerFactory, comparator, THRESHOLD_FOR_IN_PLACE_SORTING));
}
 

public TempBarrier(AbstractInvokable owner, MutableObjectIterator<T> input, TypeSerializerFactory<T> serializerFactory,
		MemoryManager memManager, IOManager ioManager, int numPages) throws MemoryAllocationException
{
	this.serializer = serializerFactory.getSerializer();
	this.memManager = memManager;
	
	this.memory = new ArrayList<MemorySegment>(numPages);
	memManager.allocatePages(owner, this.memory, numPages);
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(this.memory), memManager.getPageSize());
	this.tempWriter = new TempWritingThread(input, serializerFactory.getSerializer(), this.buffer);
}
 

public CombiningSpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
		AbstractInvokable parentTask, MemoryManager memManager, IOManager ioManager, 
		TypeSerializer<E> serializer, TypeComparator<E> comparator, 
		List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxNumFileHandles,
		boolean objectReuseEnabled)
{
	super(exceptionHandler, queues, parentTask, memManager, ioManager, serializer, comparator, 
		sortReadMemory, writeMemory, maxNumFileHandles);
	
	this.comparator2 = comparator.duplicate();
	this.objectReuseEnabled = objectReuseEnabled;
}
 

ResultPartitionWithCountDownLatch(
		String owningTaskName,
		TaskActions taskActions,
		JobID jobId,
		ResultPartitionID partitionId,
		ResultPartitionType partitionType,
		int numberOfSubpartitions,
		int numTargetKeyGroups,
		ResultPartitionManager partitionManager,
		ResultPartitionConsumableNotifier partitionConsumableNotifier,
		IOManager ioManager,
		boolean sendScheduleOrUpdateConsumersMessage,
		CountDownLatch blockLatch,
		CountDownLatch doneLatch) {
	super(
		owningTaskName,
		taskActions,
		jobId,
		partitionId,
		partitionType,
		numberOfSubpartitions,
		numTargetKeyGroups,
		partitionManager,
		partitionConsumableNotifier,
		ioManager,
		sendScheduleOrUpdateConsumersMessage);
	this.blockLatch = Preconditions.checkNotNull(blockLatch);
	this.doneLatch = Preconditions.checkNotNull(doneLatch);
}
 

public UnilateralSortMerger(MemoryManager memoryManager, List<MemorySegment> memory,
		IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask, 
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		int numSortBuffers, int maxNumFileHandles,
		float startSpillingFraction, boolean handleLargeRecords, boolean objectReuseEnabled)
throws IOException
{
	this(memoryManager, memory, ioManager, input, parentTask, serializerFactory, comparator,
		numSortBuffers, maxNumFileHandles, startSpillingFraction, false, handleLargeRecords,
		objectReuseEnabled);
}
 

@Test
public void testExecutionFailsInNetworkRegistration() throws Exception {
	// mock a network manager that rejects registration
	final ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
	final ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	final PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	final TaskEventDispatcher taskEventDispatcher = mock(TaskEventDispatcher.class);

	final NetworkEnvironment network = mock(NetworkEnvironment.class);
	when(network.getResultPartitionManager()).thenReturn(partitionManager);
	when(network.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC);
	when(network.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);
	doThrow(new RuntimeException("buffers")).when(network).registerTask(any(Task.class));

	final QueuedNoOpTaskManagerActions taskManagerActions = new QueuedNoOpTaskManagerActions();
	final Task task = new TaskBuilder()
		.setTaskManagerActions(taskManagerActions)
		.setConsumableNotifier(consumableNotifier)
		.setPartitionProducerStateChecker(partitionProducerStateChecker)
		.setNetworkEnvironment(network)
		.build();

	// should fail
	task.run();

	// verify final state
	assertEquals(ExecutionState.FAILED, task.getExecutionState());
	assertTrue(task.isCanceledOrFailed());
	assertTrue(task.getFailureCause().getMessage().contains("buffers"));

	taskManagerActions.validateListenerMessage(
		ExecutionState.FAILED, task, new RuntimeException("buffers"));
}
 

/**
 * Creates the spilling thread.
 * @param exceptionHandler  The exception handler to call for all exceptions.
 * @param queues            The circularQueues used to pass buffers between the threads.
 * @param ioManager         The I/O manager used to instantiate readers and writers from.
 * @param serializer
 * @param comparator
 */
public SpillingThread(ExceptionHandler<IOException> exceptionHandler,
		CircularQueues queues, IOManager ioManager,
		BinaryRowDataSerializer serializer, RecordComparator comparator) {
	super(exceptionHandler, "SortMerger spilling thread", queues);
	this.ioManager = ioManager;
	this.serializer = serializer;
	this.comparator = comparator;
}
 

@Override
protected ThreadBase<E> getSpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
	AbstractInvokable parentTask, MemoryManager memoryManager, IOManager ioManager, 
	TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
	List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxFileHandles)
{
	return new CombiningSpillingThread(exceptionHandler, queues, parentTask,
			memoryManager, ioManager, serializerFactory.getSerializer(),
			comparator, sortReadMemory, writeMemory, maxFileHandles, objectReuseEnabled);
}
 

public ReusingBuildSecondHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V1, V2> pairComparator,
		MemoryManager memManager,
		IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	this.memManager = memManager;
	this.firstInput = firstInput;
	this.secondInput = secondInput;
	this.probeSideSerializer = serializer1;

	if(useBitmapFilters && probeSideOuterJoin) {
		throw new IllegalArgumentException("Bitmap filter may not be activated for joining with empty build side");
	}
	this.probeSideOuterJoin = probeSideOuterJoin;
	this.buildSideOuterJoin = buildSideOuterJoin;
	
	this.nextBuildSideObject = serializer2.createInstance();
	this.tempBuildSideRecord = serializer2.createInstance();

	this.hashJoin = getHashJoin(serializer2, comparator2, serializer1, comparator1, pairComparator,
		memManager, ioManager, ownerTask, memoryFraction, useBitmapFilters);
}
 

public AbstractMergeIterator(MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2,
							TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
							TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
							TypePairComparator<T1, T2> pairComparator,
							MemoryManager memoryManager,
							IOManager ioManager,
							int numMemoryPages,
							AbstractInvokable parentTask) throws MemoryAllocationException {
	if (numMemoryPages < 2) {
		throw new IllegalArgumentException("Merger needs at least 2 memory pages.");
	}

	this.pairComparator = pairComparator;
	this.serializer1 = serializer1;
	this.serializer2 = serializer2;

	this.memoryManager = memoryManager;
	this.ioManager = ioManager;

	this.iterator1 = createKeyGroupedIterator(input1, serializer1, comparator1.duplicate());
	this.iterator2 = createKeyGroupedIterator(input2, serializer2, comparator2.duplicate());

	final int numPagesForSpiller = numMemoryPages > 20 ? 2 : 1;
	this.blockIt = new NonReusingBlockResettableIterator<>(this.memoryManager, this.serializer2,
			(numMemoryPages - numPagesForSpiller), parentTask);
	this.memoryForSpillingIterator = memoryManager.allocatePages(parentTask, numPagesForSpiller);
}
 

/**
 * Spills this partition to disk and sets it up such that it continues spilling records that are added to
 * it. The spilling process must free at least one buffer, either in the partition's record buffers, or in
 * the memory segments for overflow buckets.
 * The partition immediately takes back one buffer to use it for further spilling.
 * 
 * @param target The list to which memory segments from overflow buckets are added.
 * @param ioAccess The I/O manager to be used to create a writer to disk.
 * @param targetChannel The id of the target channel for this partition.
 * @return The number of buffers that were freed by spilling this partition.
 * @throws IOException Thrown, if the writing failed.
 */
public int spillPartition(List<MemorySegment> target, IOManager ioAccess, FileIOChannel.ID targetChannel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue)
throws IOException
{
	// sanity checks
	if (!isInMemory()) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition that has already been spilled.");
	}
	if (getNumOccupiedMemorySegments() < 2) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
			"Request to spill a partition with less than two buffers.");
	}
	
	// return the memory from the overflow segments
	for (int i = 0; i < this.numOverflowSegments; i++) {
		target.add(this.overflowSegments[i]);
	}
	this.overflowSegments = null;
	this.numOverflowSegments = 0;
	this.nextOverflowBucket = 0;
	
	// create the channel block writer and spill the current buffers
	// that keep the build side buffers current block, as it is most likely not full, yet
	// we return the number of blocks that become available
	this.buildSideChannel = ioAccess.createBlockChannelWriter(targetChannel, bufferReturnQueue);
	return this.buildSideWriteBuffer.spill(this.buildSideChannel);
}
 

@Test
public void testCloseAndDeleteOutputView() {
	final IOManager ioManager = new IOManagerAsync();
	try {
		MemoryManager memMan = new MemoryManager(4 * 16*1024, 1, 16*1024, MemoryType.HEAP, true);
		List<MemorySegment> memory = new ArrayList<MemorySegment>();
		memMan.allocatePages(new DummyInvokable(), memory, 4);
		
		FileIOChannel.ID channel = ioManager.createChannel();
		BlockChannelWriter<MemorySegment> writer = ioManager.createBlockChannelWriter(channel);
		
		FileChannelOutputView out = new FileChannelOutputView(writer, memMan, memory, memMan.getPageSize());
		new StringValue("Some test text").write(out);
		
		// close for the first time, make sure all memory returns
		out.close();
		assertTrue(memMan.verifyEmpty());
		
		// close again, should not cause an exception
		out.close();
		
		// delete, make sure file is removed
		out.closeAndDelete();
		assertFalse(new File(channel.getPath()).exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
	finally {
		ioManager.shutdown();
	}
}
 

/**
 * This tests the case where no additional partition buffers are used at the point when spilling
 * is triggered, testing that overflow bucket buffers are taken into account when deciding which
 * partition to spill.
 */
@Test
public void testSpillingFreesOnlyOverflowSegments() {
	final TypeSerializer<ByteValue> serializer = ByteValueSerializer.INSTANCE;
	final TypeComparator<ByteValue> buildComparator = new ValueComparator<>(true, ByteValue.class);
	final TypeComparator<ByteValue> probeComparator = new ValueComparator<>(true, ByteValue.class);
	
	@SuppressWarnings("unchecked")
	final TypePairComparator<ByteValue, ByteValue> pairComparator = Mockito.mock(TypePairComparator.class);
	
	try (final IOManager ioMan = new IOManagerAsync()) {
		final int pageSize = 32*1024;
		final int numSegments = 34;

		List<MemorySegment> memory = getMemory(numSegments, pageSize);

		MutableHashTable<ByteValue, ByteValue> table = new MutableHashTable<>(
				serializer, serializer, buildComparator, probeComparator,
				pairComparator, memory, ioMan, 1, false);

		table.open(new ByteValueIterator(100000000), new ByteValueIterator(1));
		
		table.close();
		
		checkNoTempFilesRemain(ioMan);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

public UnilateralSortMerger(MemoryManager memoryManager, IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask,
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		double memoryFraction, int numSortBuffers, int maxNumFileHandles,
		float startSpillingFraction, boolean handleLargeRecords, boolean objectReuseEnabled)
throws IOException, MemoryAllocationException
{
	this(memoryManager, ioManager, input, parentTask, serializerFactory, comparator,
		memoryFraction, numSortBuffers, maxNumFileHandles, startSpillingFraction, false, handleLargeRecords,
		objectReuseEnabled);
}
 

protected abstract JoinTaskIterator<IT1, IT2, OT> getReusingOuterJoinIterator(
		DriverStrategy driverStrategy,
		MutableObjectIterator<IT1> in1,
		MutableObjectIterator<IT2> in2,
		TypeSerializer<IT1> serializer1,
		TypeComparator<IT1> comparator1,
		TypeSerializer<IT2> serializer2,
		TypeComparator<IT2> comparator2,
		TypePairComparatorFactory<IT1, IT2> pairComparatorFactory,
		MemoryManager memoryManager,
		IOManager ioManager,
		double driverMemFraction
) throws Exception;
 

/**
 * This tests the case where no additional partition buffers are used at the point when spilling
 * is triggered, testing that overflow bucket buffers are taken into account when deciding which
 * partition to spill.
 */
@Test
public void testSpillingFreesOnlyOverflowSegments() {
	final IOManager ioMan = new IOManagerAsync();
	
	final TypeSerializer<ByteValue> serializer = ByteValueSerializer.INSTANCE;
	final TypeComparator<ByteValue> buildComparator = new ValueComparator<>(true, ByteValue.class);
	final TypeComparator<ByteValue> probeComparator = new ValueComparator<>(true, ByteValue.class);
	
	@SuppressWarnings("unchecked")
	final TypePairComparator<ByteValue, ByteValue> pairComparator = Mockito.mock(TypePairComparator.class);
	
	try {
		final int pageSize = 32*1024;
		final int numSegments = 34;

		List<MemorySegment> memory = getMemory(numSegments, pageSize);

		MutableHashTable<ByteValue, ByteValue> table = new MutableHashTable<>(
				serializer, serializer, buildComparator, probeComparator,
				pairComparator, memory, ioMan, 1, false);

		table.open(new ByteValueIterator(100000000), new ByteValueIterator(1));
		
		table.close();
		
		checkNoTempFilesRemain(ioMan);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
	finally {
		ioMan.shutdown();
	}
}