Java Code Examples for org.apache.flink.api.java.tuple.Tuple2#setField()

@Override
public void combine(Iterable<Tuple2<Integer, Integer>> values, Collector<Tuple2<Integer, Integer>> out) {
	ArrayList<Tuple2<Integer, Integer>> valueList = new ArrayList<>();
	for (Tuple2<Integer, Integer> next : values) {
		valueList.add(next);
	}

	int count = 0;
	Tuple2<Integer, Integer> rec = new Tuple2<>();
	for (Tuple2<Integer, Integer> tuple : valueList) {
		rec.setField(tuple.f0, 0);
		count += tuple.f1;
	}
	rec.setField(count, 1);
	out.collect(rec);
}
 

@Override
public void run(SourceContext<OUT> ctx) throws Exception {
	VectorLoader vectorLoader = new VectorLoader(root);
	while (running && !indexesToEmit.isEmpty()) {
		Tuple2<Integer, Integer> indexToEmit = indexesToEmit.peek();
		ArrowRecordBatch arrowRecordBatch = loadBatch(indexToEmit.f0);
		vectorLoader.load(arrowRecordBatch);
		arrowRecordBatch.close();

		ArrowReader<OUT> arrowReader = createArrowReader(root);
		int rowCount = root.getRowCount();
		int nextRowId = indexToEmit.f1;
		while (nextRowId < rowCount) {
			OUT element = arrowReader.read(nextRowId);
			synchronized (ctx.getCheckpointLock()) {
				ctx.collect(element);
				indexToEmit.setField(++nextRowId, 1);
			}
		}

		synchronized (ctx.getCheckpointLock()) {
			indexesToEmit.pop();
		}
	}
}
 

@Override
public void combine(Iterable<Tuple2<Integer, Integer>> values, Collector<Tuple2<Integer, Integer>> out) {
	ArrayList<Tuple2<Integer, Integer>> valueList = new ArrayList<>();
	for (Tuple2<Integer, Integer> next : values) {
		valueList.add(next);
	}

	int count = 0;
	Tuple2<Integer, Integer> rec = new Tuple2<>();
	for (Tuple2<Integer, Integer> tuple : valueList) {
		rec.setField(tuple.f0, 0);
		count += tuple.f1;
	}
	rec.setField(count, 1);
	out.collect(rec);
}
 

@Override
public void combine(Iterable<Tuple2<Integer, Integer>> values, Collector<Tuple2<Integer, Integer>> out) {
	ArrayList<Tuple2<Integer, Integer>> valueList = new ArrayList<>();
	for (Tuple2<Integer, Integer> next : values) {
		valueList.add(next);
	}

	int count = 0;
	Tuple2<Integer, Integer> rec = new Tuple2<>();
	for (Tuple2<Integer, Integer> tuple : valueList) {
		rec.setField(tuple.f0, 0);
		count += tuple.f1;
	}
	rec.setField(count, 1);
	out.collect(rec);
}
 

private MutableObjectIterator<Tuple2<Integer, String>> newIterator(final int[] keys, final String[] values) {
	
	return new MutableObjectIterator<Tuple2<Integer, String>>() {
		
		private int key = 0;
		private String value = new String();
		
		private int current = 0;

		@Override
		public Tuple2<Integer, String> next(Tuple2<Integer, String> reuse) {
			if (current < keys.length) {
				key = keys[current];
				value = values[current];
				current++;
				reuse.setField(key, 0);
				reuse.setField(value, 1);
				return reuse;
			}
			else {
				return null;
			}
		}

		@Override
		public Tuple2<Integer, String> next() {
			if (current < keys.length) {
				Tuple2<Integer, String> result = new Tuple2<>(keys[current], values[current]);
				current++;
				return result;
			}
			else {
				return null;
			}
		}
	};
}
 

private MutableObjectIterator<Tuple2<Integer, String>> newIterator(final int[] keys, final String[] values) {
	
	return new MutableObjectIterator<Tuple2<Integer, String>>() {
		
		private int key = 0;
		private String value = new String();
		
		private int current = 0;

		@Override
		public Tuple2<Integer, String> next(Tuple2<Integer, String> reuse) {
			if (current < keys.length) {
				key = keys[current];
				value = values[current];
				current++;
				reuse.setField(key, 0);
				reuse.setField(value, 1);
				return reuse;
			}
			else {
				return null;
			}
		}

		@Override
		public Tuple2<Integer, String> next() {
			if (current < keys.length) {
				Tuple2<Integer, String> result = new Tuple2<>(keys[current], values[current]);
				current++;
				return result;
			}
			else {
				return null;
			}
		}
	};
}
 

@Override
public Tuple2<Long, Long> join(Tuple2<Long, Long> edge,
		Tuple2<Long, Long> vertexWithCompId) throws Exception {

	vertexWithCompId.setField(edge.f1, 0);
	return vertexWithCompId;
}
 

@Override
public Tuple2<Integer, Integer> next(Tuple2<Integer, Integer> reuse) {
	if (this.numLeft > 0) {
		this.numLeft--;
		reuse.setField(this.key, 0);
		reuse.setField(this.value, 1);
		return reuse;
	} else {
		return null;
	}
}
 

@Test
public void testCombineSpilling() throws Exception {
	int noKeys = 100;
	int noKeyCnt = 10000;

	TestData.MockTuple2Reader<Tuple2<Integer, Integer>> reader = TestData.getIntIntTupleReader();

	LOG.debug("initializing sortmerger");
	
	TestCountCombiner comb = new TestCountCombiner();
	
	Sorter<Tuple2<Integer, Integer>> merger = new CombiningUnilateralSortMerger<>(comb,
			this.memoryManager, this.ioManager, reader, this.parentTask, this.serializerFactory2, this.comparator2,
			0.01, 64, 0.005f, true /* use large record handler */, true);

	final Tuple2<Integer, Integer> rec = new Tuple2<>();
	rec.setField(1, 1);
	
	for (int i = 0; i < noKeyCnt; i++) {
		for (int j = 0; j < noKeys; j++) {
			rec.setField(j, 0);
			reader.emit(rec);
		}
	}
	reader.close();
	
	MutableObjectIterator<Tuple2<Integer, Integer>> iterator = merger.getIterator();

	Iterator<Integer> result = getReducingIterator(iterator, serializerFactory2.getSerializer(), comparator2.duplicate());
	while (result.hasNext()) {
		Assert.assertEquals(noKeyCnt, result.next().intValue());
	}
	
	merger.close();
	
	// if the combiner was opened, it must have been closed
	Assert.assertTrue(comb.opened == comb.closed);
}
 

@Test
public void testCombine() throws Exception
{
	int noKeys = 100;
	int noKeyCnt = 10000;

	TestData.MockTuple2Reader<Tuple2<Integer, Integer>> reader = TestData.getIntIntTupleReader();

	LOG.debug("initializing sortmerger");
	
	TestCountCombiner comb = new TestCountCombiner();
	
	Sorter<Tuple2<Integer, Integer>> merger = new CombiningUnilateralSortMerger<>(comb,
			this.memoryManager, this.ioManager, reader, this.parentTask, this.serializerFactory2, this.comparator2,
			0.25, 64, 0.7f, true /* use large record handler */, false);

	final Tuple2<Integer, Integer> rec = new Tuple2<>();
	rec.setField(1, 1);
	
	for (int i = 0; i < noKeyCnt; i++) {
		for (int j = 0; j < noKeys; j++) {
			rec.setField(j, 0);
			reader.emit(rec);
		}
	}
	reader.close();
	
	MutableObjectIterator<Tuple2<Integer, Integer>> iterator = merger.getIterator();

	Iterator<Integer> result = getReducingIterator(iterator, serializerFactory2.getSerializer(), comparator2.duplicate());
	while (result.hasNext()) {
		Assert.assertEquals(noKeyCnt, result.next().intValue());
	}
	
	merger.close();
	
	// if the combiner was opened, it must have been closed
	Assert.assertTrue(comb.opened == comb.closed);
}
 

@Override
public Tuple2<Long, Long> join(Tuple2<Long, Long> edge,
		Tuple2<Long, Long> vertexWithCompId) throws Exception {

	vertexWithCompId.setField(edge.f1, 0);
	return vertexWithCompId;
}
 

private MutableObjectIterator<Tuple2<Integer, String>> newIterator(final int[] keys, final String[] values) {
	
	return new MutableObjectIterator<Tuple2<Integer, String>>() {
		
		private int key = 0;
		private String value = new String();
		
		private int current = 0;

		@Override
		public Tuple2<Integer, String> next(Tuple2<Integer, String> reuse) {
			if (current < keys.length) {
				key = keys[current];
				value = values[current];
				current++;
				reuse.setField(key, 0);
				reuse.setField(value, 1);
				return reuse;
			}
			else {
				return null;
			}
		}

		@Override
		public Tuple2<Integer, String> next() {
			if (current < keys.length) {
				Tuple2<Integer, String> result = new Tuple2<>(keys[current], values[current]);
				current++;
				return result;
			}
			else {
				return null;
			}
		}
	};
}
 

@Test
public void testCombineSpillingDisableObjectReuse() throws Exception {
	int noKeys = 100;
	int noKeyCnt = 10000;

	TestData.MockTuple2Reader<Tuple2<Integer, Integer>> reader = TestData.getIntIntTupleReader();

	LOG.debug("initializing sortmerger");

	MaterializedCountCombiner comb = new MaterializedCountCombiner();

	// set maxNumFileHandles = 2 to trigger multiple channel merging
	Sorter<Tuple2<Integer, Integer>> merger = new CombiningUnilateralSortMerger<>(comb,
			this.memoryManager, this.ioManager, reader, this.parentTask, this.serializerFactory2, this.comparator2,
			0.01, 2, 0.005f, true /* use large record handler */, false);

	final Tuple2<Integer, Integer> rec = new Tuple2<>();

	for (int i = 0; i < noKeyCnt; i++) {
		rec.setField(i, 0);
		for (int j = 0; j < noKeys; j++) {
			rec.setField(j, 1);
			reader.emit(rec);
		}
	}
	reader.close();

	MutableObjectIterator<Tuple2<Integer, Integer>> iterator = merger.getIterator();
	Iterator<Integer> result = getReducingIterator(iterator, serializerFactory2.getSerializer(), comparator2.duplicate());
	while (result.hasNext()) {
		Assert.assertEquals(4950, result.next().intValue());
	}

	merger.close();
}
 

@Test
public void testCombineSpilling() throws Exception {
	int noKeys = 100;
	int noKeyCnt = 10000;

	TestData.MockTuple2Reader<Tuple2<Integer, Integer>> reader = TestData.getIntIntTupleReader();

	LOG.debug("initializing sortmerger");
	
	TestCountCombiner comb = new TestCountCombiner();
	
	Sorter<Tuple2<Integer, Integer>> merger = new CombiningUnilateralSortMerger<>(comb,
			this.memoryManager, this.ioManager, reader, this.parentTask, this.serializerFactory2, this.comparator2,
			0.01, 64, 0.005f, true /* use large record handler */, true);

	final Tuple2<Integer, Integer> rec = new Tuple2<>();
	rec.setField(1, 1);
	
	for (int i = 0; i < noKeyCnt; i++) {
		for (int j = 0; j < noKeys; j++) {
			rec.setField(j, 0);
			reader.emit(rec);
		}
	}
	reader.close();
	
	MutableObjectIterator<Tuple2<Integer, Integer>> iterator = merger.getIterator();

	Iterator<Integer> result = getReducingIterator(iterator, serializerFactory2.getSerializer(), comparator2.duplicate());
	while (result.hasNext()) {
		Assert.assertEquals(noKeyCnt, result.next().intValue());
	}
	
	merger.close();
	
	// if the combiner was opened, it must have been closed
	Assert.assertTrue(comb.opened == comb.closed);
}
 

@Override
public void combine(Iterable<Tuple2<Integer, Integer>> values, Collector<Tuple2<Integer, Integer>> out) {
	Tuple2<Integer, Integer> rec = new Tuple2<>();
	int cnt = 0;
	for (Tuple2<Integer, Integer> next : values) {
		rec = next;
		cnt += rec.f1;
	}
	
	this.count = cnt;
	rec.setField(this.count, 1);
	out.collect(rec);
}
 

@Override
public Tuple2<Integer, Integer> map(Tuple2<Integer, Integer> value) throws Exception {
	Tuple2<Integer, Integer> t = new Tuple2<Integer, Integer>();
	t.setField(value.f1, 0);
	t.setField(value.getField(0), 1);
	return t;
}
 

@Override
public Tuple2<Integer, Integer> map(Tuple2<Integer, Integer> value) throws Exception {
	Integer i = value.f0;
	value.setField(value.f1, 0);
	value.setField(i, 1);
	return value;
}
 

@Test
public void testSortAndValidate() throws Exception
{
	final Hashtable<Integer, Integer> countTable = new Hashtable<>(KEY_MAX);
	for (int i = 1; i <= KEY_MAX; i++) {
		countTable.put(i, 0);
	}

	// comparator
	final TypeComparator<Integer> keyComparator = new IntComparator(true);

	// reader
	TestData.MockTuple2Reader<Tuple2<Integer, String>> reader = TestData.getIntStringTupleReader();

	// merge iterator
	LOG.debug("initializing sortmerger");
	
	TestCountCombiner2 comb = new TestCountCombiner2();
	
	Sorter<Tuple2<Integer, String>> merger = new CombiningUnilateralSortMerger<>(comb,
			this.memoryManager, this.ioManager, reader, this.parentTask, this.serializerFactory1, this.comparator1,
			0.25, 2, 0.7f, true /* use large record handler */, false);

	// emit data
	LOG.debug("emitting data");
	TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.FIX_LENGTH);
	Tuple2<Integer, String> rec = new Tuple2<>();
	
	for (int i = 0; i < NUM_PAIRS; i++) {
		Assert.assertTrue((rec = generator.next(rec)) != null);
		final Integer key = rec.f0;
		rec.setField("1", 1);
		reader.emit(rec);
		
		countTable.put(key, countTable.get(key) + 1);
	}
	reader.close();

	// check order
	MutableObjectIterator<Tuple2<Integer, String>> iterator = merger.getIterator();
	
	LOG.debug("checking results");
	
	Tuple2<Integer, String> rec1 = new Tuple2<>();
	Tuple2<Integer, String> rec2 = new Tuple2<>();
	
	Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
	countTable.put(rec1.f0, countTable.get(rec1.f0) - (Integer.parseInt(rec1.f1)));

	while ((rec2 = iterator.next(rec2)) != null) {
		int k1 = rec1.f0;
		int k2 = rec2.f0;
		
		Assert.assertTrue(keyComparator.compare(k1, k2) <= 0); 
		countTable.put(k2, countTable.get(k2) - (Integer.parseInt(rec2.f1)));
		
		rec1 = rec2;
	}

	for (Integer cnt : countTable.values()) {
		Assert.assertTrue(cnt == 0);
	}
	
	merger.close();
	
	// if the combiner was opened, it must have been closed
	Assert.assertTrue(comb.opened == comb.closed);
}
 

@Test
public void testSortAndValidate() throws Exception
{
	final Hashtable<Integer, Integer> countTable = new Hashtable<>(KEY_MAX);
	for (int i = 1; i <= KEY_MAX; i++) {
		countTable.put(i, 0);
	}

	// comparator
	final TypeComparator<Integer> keyComparator = new IntComparator(true);

	// reader
	TestData.MockTuple2Reader<Tuple2<Integer, String>> reader = TestData.getIntStringTupleReader();

	// merge iterator
	LOG.debug("initializing sortmerger");
	
	TestCountCombiner2 comb = new TestCountCombiner2();
	
	Sorter<Tuple2<Integer, String>> merger = new CombiningUnilateralSortMerger<>(comb,
			this.memoryManager, this.ioManager, reader, this.parentTask, this.serializerFactory1, this.comparator1,
			0.25, 2, 0.7f, true /* use large record handler */, false);

	// emit data
	LOG.debug("emitting data");
	TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.FIX_LENGTH);
	Tuple2<Integer, String> rec = new Tuple2<>();
	
	for (int i = 0; i < NUM_PAIRS; i++) {
		Assert.assertTrue((rec = generator.next(rec)) != null);
		final Integer key = rec.f0;
		rec.setField("1", 1);
		reader.emit(rec);
		
		countTable.put(key, countTable.get(key) + 1);
	}
	reader.close();

	// check order
	MutableObjectIterator<Tuple2<Integer, String>> iterator = merger.getIterator();
	
	LOG.debug("checking results");
	
	Tuple2<Integer, String> rec1 = new Tuple2<>();
	Tuple2<Integer, String> rec2 = new Tuple2<>();
	
	Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
	countTable.put(rec1.f0, countTable.get(rec1.f0) - (Integer.parseInt(rec1.f1)));

	while ((rec2 = iterator.next(rec2)) != null) {
		int k1 = rec1.f0;
		int k2 = rec2.f0;
		
		Assert.assertTrue(keyComparator.compare(k1, k2) <= 0); 
		countTable.put(k2, countTable.get(k2) - (Integer.parseInt(rec2.f1)));
		
		rec1 = rec2;
	}

	for (Integer cnt : countTable.values()) {
		Assert.assertTrue(cnt == 0);
	}
	
	merger.close();
	
	// if the combiner was opened, it must have been closed
	Assert.assertTrue(comb.opened == comb.closed);
}
 

public Tuple2<Long, Long> foldEdges(Tuple2<Long, Long> accum, Long id, Long neighborID, Long edgeValue) {
	accum.setField(id, 0);
	accum.setField(accum.f1 + edgeValue, 1);
	return accum;
}