org.apache.flink.core.io.GenericInputSplit Java Examples

@Override
public void open(GenericInputSplit split) throws IOException {
	this.partitionId = split.getSplitNumber();
	// total number of partitions
	int numPartitions = split.getTotalNumberOfSplits();

	// ensure even distribution of records and keys
	Preconditions.checkArgument(
		numRecords % numPartitions == 0,
		"Records cannot be evenly distributed among partitions");
	Preconditions.checkArgument(
		numKeys % numPartitions == 0,
		"Keys cannot be evenly distributed among partitions");

	this.recordsPerPartition = numRecords / numPartitions;
	this.keysPerPartition = numKeys / numPartitions;

	this.recordCnt = 0;
}
 

@Override
public void open(GenericInputSplit split) throws IOException {
	this.partitionId = split.getSplitNumber();
	// total number of partitions
	int numPartitions = split.getTotalNumberOfSplits();

	// ensure even distribution of records and keys
	Preconditions.checkArgument(
		numRecords % numPartitions == 0,
		"Records cannot be evenly distributed among partitions");
	Preconditions.checkArgument(
		numKeys % numPartitions == 0,
		"Keys cannot be evenly distributed among partitions");

	this.recordsPerPartition = numRecords / numPartitions;
	this.keysPerPartition = numKeys / numPartitions;

	this.recordCnt = 0;
}
 

@Test
public void testReplicatedSourceToJoin() throws Exception {
	/*
	 * Test replicated source going into join
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple1<Long>> source1 = env.createInput(new ReplicatingInputFormat<Long, GenericInputSplit>
			(new ParallelIteratorInputFormat<Long>(new NumberSequenceIterator(0L, 1000L))), BasicTypeInfo.LONG_TYPE_INFO)
			.map(new ToTuple());
	DataSet<Tuple1<Long>> source2 = env.generateSequence(0L, 1000L).map(new ToTuple());

	DataSet<Tuple> pairs = source1.join(source2).where(0).equalTo(0)
			.projectFirst(0)
			.sum(0);

	List<Tuple> result = pairs.collect();

	String expectedResult = "(500500)";

	compareResultAsText(result, expectedResult);
}
 

@Test
public void testSerialSplitAssignment() {
	try {
		final int NUM_SPLITS = 50;
		
		Set<InputSplit> splits = new HashSet<InputSplit>();
		for (int i = 0; i < NUM_SPLITS; i++) {
			splits.add(new GenericInputSplit(i, NUM_SPLITS));
		}
		
		DefaultInputSplitAssigner ia = new DefaultInputSplitAssigner(splits);
		InputSplit is = null;
		while ((is = ia.getNextInputSplit("", 0)) != null) {
			assertTrue(splits.remove(is));
		}
		
		assertTrue(splits.isEmpty());
		assertNull(ia.getNextInputSplit("", 0));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testSerialSplitAssignment() {
	try {
		final int NUM_SPLITS = 50;
		
		Set<InputSplit> splits = new HashSet<InputSplit>();
		for (int i = 0; i < NUM_SPLITS; i++) {
			splits.add(new GenericInputSplit(i, NUM_SPLITS));
		}
		
		DefaultInputSplitAssigner ia = new DefaultInputSplitAssigner(splits);
		InputSplit is = null;
		while ((is = ia.getNextInputSplit("", 0)) != null) {
			assertTrue(splits.remove(is));
		}
		
		assertTrue(splits.isEmpty());
		assertNull(ia.getNextInputSplit("", 0));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Override
public void open(GenericInputSplit split) throws IOException {
	this.partitionId = split.getSplitNumber();
	// total number of partitions
	int numPartitions = split.getTotalNumberOfSplits();

	// ensure even distribution of records and keys
	Preconditions.checkArgument(
		numRecords % numPartitions == 0,
		"Records cannot be evenly distributed among partitions");
	Preconditions.checkArgument(
		numKeys % numPartitions == 0,
		"Keys cannot be evenly distributed among partitions");

	this.recordsPerPartition = numRecords / numPartitions;
	this.keysPerPartition = numKeys / numPartitions;

	this.recordCnt = 0;
}
 

@Test
public void testReplicatedSourceToJoin() throws Exception {
	/*
	 * Test replicated source going into join
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple1<Long>> source1 = env.createInput(new ReplicatingInputFormat<Long, GenericInputSplit>
			(new ParallelIteratorInputFormat<Long>(new NumberSequenceIterator(0L, 1000L))), BasicTypeInfo.LONG_TYPE_INFO)
			.map(new ToTuple());
	DataSet<Tuple1<Long>> source2 = env.generateSequence(0L, 1000L).map(new ToTuple());

	DataSet<Tuple> pairs = source1.join(source2).where(0).equalTo(0)
			.projectFirst(0)
			.sum(0);

	List<Tuple> result = pairs.collect();

	String expectedResult = "(500500)";

	compareResultAsText(result, expectedResult);
}
 

@Test
public void testReplicatedSourceToJoin() throws Exception {
	/*
	 * Test replicated source going into join
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple1<Long>> source1 = env.createInput(new ReplicatingInputFormat<Long, GenericInputSplit>
			(new ParallelIteratorInputFormat<Long>(new NumberSequenceIterator(0L, 1000L))), BasicTypeInfo.LONG_TYPE_INFO)
			.map(new ToTuple());
	DataSet<Tuple1<Long>> source2 = env.generateSequence(0L, 1000L).map(new ToTuple());

	DataSet<Tuple> pairs = source1.join(source2).where(0).equalTo(0)
			.projectFirst(0)
			.sum(0);

	List<Tuple> result = pairs.collect();

	String expectedResult = "(500500)";

	compareResultAsText(result, expectedResult);
}
 

@Test
public void testSerialSplitAssignment() {
	try {
		final int NUM_SPLITS = 50;
		
		Set<InputSplit> splits = new HashSet<InputSplit>();
		for (int i = 0; i < NUM_SPLITS; i++) {
			splits.add(new GenericInputSplit(i, NUM_SPLITS));
		}
		
		DefaultInputSplitAssigner ia = new DefaultInputSplitAssigner(splits);
		InputSplit is = null;
		while ((is = ia.getNextInputSplit("", 0)) != null) {
			assertTrue(splits.remove(is));
		}
		
		assertTrue(splits.isEmpty());
		assertNull(ia.getNextInputSplit("", 0));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Override
public GenericInputSplit[] createInputSplits(int numSplits) throws IOException {
	if (numSplits < 1) {
		throw new IllegalArgumentException("Number of input splits has to be at least 1.");
	}

	numSplits = (this instanceof NonParallelInput) ? 1 : numSplits;
	GenericInputSplit[] splits = new GenericInputSplit[numSplits];
	for (int i = 0; i < splits.length; i++) {
		splits[i] = new GenericInputSplit(i, numSplits);
	}
	return splits;
}
 

@Override
public GenericInputSplit[] createInputSplits(int minNumSplits) {

	GenericInputSplit[] splits = new GenericInputSplit[minNumSplits];
	for (int i = 0; i < minNumSplits; i++) {
		splits[i] = new GenericInputSplit(i, minNumSplits);
	}
	return splits;
}
 

@Override
public GenericInputSplit[] createInputSplits(int numSplits) throws IOException {
	if (numSplits < 1) {
		throw new IllegalArgumentException("Number of input splits has to be at least 1.");
	}

	numSplits = (this instanceof NonParallelInput) ? 1 : numSplits;
	GenericInputSplit[] splits = new GenericInputSplit[numSplits];
	for (int i = 0; i < splits.length; i++) {
		splits[i] = new GenericInputSplit(i, numSplits);
	}
	return splits;
}
 

@Override
public GenericInputSplit[] createInputSplits(int numSplits) throws IOException {
	if (numSplits < 1) {
		throw new IllegalArgumentException("Number of input splits has to be at least 1.");
	}

	numSplits = (this instanceof NonParallelInput) ? 1 : numSplits;
	GenericInputSplit[] splits = new GenericInputSplit[numSplits];
	for (int i = 0; i < splits.length; i++) {
		splits[i] = new GenericInputSplit(i, numSplits);
	}
	return splits;
}
 

@Override
public InputSplit[] createInputSplits(int minNumSplits) throws IOException {
	if (parameterValues == null) {
		return new GenericInputSplit[]{new GenericInputSplit(0, 1)};
	}
	GenericInputSplit[] ret = new GenericInputSplit[parameterValues.length];
	for (int i = 0; i < ret.length; i++) {
		ret[i] = new GenericInputSplit(i, ret.length);
	}
	return ret;
}
 

@Override
public GenericInputSplit[] createInputSplits(int numSplits) throws IOException {
	if (numSplits < 1) {
		throw new IllegalArgumentException("Number of input splits has to be at least 1.");
	}

	numSplits = (this instanceof NonParallelInput) ? 1 : numSplits;
	GenericInputSplit[] splits = new GenericInputSplit[numSplits];
	for (int i = 0; i < splits.length; i++) {
		splits[i] = new GenericInputSplit(i, numSplits);
	}
	return splits;
}
 

@Override
public InputSplit[] createInputSplits(int minNumSplits) throws IOException {
	if (parameterValues == null) {
		return new GenericInputSplit[]{new GenericInputSplit(0, 1)};
	}
	GenericInputSplit[] ret = new GenericInputSplit[parameterValues.length];
	for (int i = 0; i < ret.length; i++) {
		ret[i] = new GenericInputSplit(i, ret.length);
	}
	return ret;
}
 

@Override
public void open(GenericInputSplit split) {
	LOG.debug("Compiling GenericInputFormat: $name \n\n Code:\n$code",
			generatedInput.getClassName(), generatedInput.getCode());
	LOG.debug("Instantiating GenericInputFormat.");

	format = generatedInput.newInstance(getRuntimeContext().getUserCodeClassLoader());
	generatedInput = null;
}
 

@Override
public GenericInputSplit[] createInputSplits(int minNumSplits) {

	GenericInputSplit[] splits = new GenericInputSplit[minNumSplits];
	for (int i = 0; i < minNumSplits; i++) {
		splits[i] = new GenericInputSplit(i, minNumSplits);
	}
	return splits;
}
 

@Override
public GenericInputSplit[] createInputSplits(int numSplits) {
	if (!isConfigured) {
		throw new IllegalStateException("InputFormat was not configured before createInputSplits was called.");
	}
	return new GenericInputSplit[] { new TestSplit(0, 1) };
}
 

@Override
public GenericInputSplit[] createInputSplits(int numSplits) throws IOException {
	if (numSplits < 1) {
		throw new IllegalArgumentException("Number of input splits has to be at least 1.");
	}

	numSplits = (this instanceof NonParallelInput) ? 1 : numSplits;
	GenericInputSplit[] splits = new GenericInputSplit[numSplits];
	for (int i = 0; i < splits.length; i++) {
		splits[i] = new GenericInputSplit(i, numSplits);
	}
	return splits;
}
 

@Override
public GenericInputSplit[] createInputSplits(int numSplits) {
	if (!isConfigured) {
		throw new IllegalStateException("InputFormat was not configured before createInputSplits was called.");
	}
	return new GenericInputSplit[] { new TestSplit(0, 1) };
}
 

@Override
public GenericInputSplit[] createInputSplits(int minNumSplits) {

	GenericInputSplit[] splits = new GenericInputSplit[minNumSplits];
	for (int i = 0; i < minNumSplits; i++) {
		splits[i] = new GenericInputSplit(i, minNumSplits);
	}
	return splits;
}
 

@Override
public InputSplit[] createInputSplits(int minNumSplits) throws IOException {
	if (parameterValues == null) {
		return new GenericInputSplit[]{new GenericInputSplit(0, 1)};
	}
	GenericInputSplit[] ret = new GenericInputSplit[parameterValues.length];
	for (int i = 0; i < ret.length; i++) {
		ret[i] = new GenericInputSplit(i, ret.length);
	}
	return ret;
}
 

@Override
public InputSplit[] createInputSplits(int minNumSplits) throws IOException {
	if (parameterValues == null) {
		return new GenericInputSplit[]{new GenericInputSplit(0, 1)};
	}
	GenericInputSplit[] ret = new GenericInputSplit[parameterValues.length];
	for (int i = 0; i < ret.length; i++) {
		ret[i] = new GenericInputSplit(i, ret.length);
	}
	return ret;
}
 

@Override
public void open(GenericInputSplit split) {
	LOG.debug("Compiling GenericInputFormat: {} \n\n Code:\n{}",
			generatedInput.getClassName(), generatedInput.getCode());
	LOG.debug("Instantiating GenericInputFormat.");

	format = generatedInput.newInstance(getRuntimeContext().getUserCodeClassLoader());
	generatedInput = null;
}
 

@Override
public GenericInputSplit[] createInputSplits(int numSplits) throws IOException {
	if (numSplits < 1) {
		throw new IllegalArgumentException("Number of input splits has to be at least 1.");
	}

	numSplits = (this instanceof NonParallelInput) ? 1 : numSplits;
	GenericInputSplit[] splits = new GenericInputSplit[numSplits];
	for (int i = 0; i < splits.length; i++) {
		splits[i] = new GenericInputSplit(i, numSplits);
	}
	return splits;
}
 

@Override
public void open(GenericInputSplit split) throws IOException {
	super.open(split);

	this.iterator = this.dataSet.iterator();
}
 

@Override
public GenericInputSplit[] createInputSplits(int numSplits) throws IOException {
	assertEquals(USER_DOP, numSplits);
	return super.createInputSplits(numSplits);
}
 

@Override
public GenericInputSplit[] createInputSplits(int numSplits) throws IOException {
	assertEquals(PARALLELISM, numSplits);
	return super.createInputSplits(numSplits);
}
 

@Override
public void open(GenericInputSplit split) throws IOException {
	super.open(split);

	this.splitIterator = this.source.getSplit(split.getSplitNumber(), split.getTotalNumberOfSplits());
}