org.apache.flink.types.IntValue Java Examples

@Test
public void testWithDirectedSimpleGraph() throws Exception {
	DataSet<Vertex<IntValue, LongValue>> inDegree = directedSimpleGraph
		.run(new VertexInDegree<IntValue, NullValue, NullValue>()
			.setIncludeZeroDegreeVertices(true));

	String expectedResult =
		"(0,0)\n" +
		"(1,3)\n" +
		"(2,1)\n" +
		"(3,2)\n" +
		"(4,1)\n" +
		"(5,0)";

	TestBaseUtils.compareResultAsText(inDegree.collect(), expectedResult);
}
 

@Test
public void testWithSimpleGraph() throws Exception {
	DataSet<Result<IntValue>> hits = new HITS<IntValue, NullValue, NullValue>(20)
		.run(directedSimpleGraph);

	List<Tuple2<Double, Double>> expectedResults = new ArrayList<>();
	expectedResults.add(Tuple2.of(0.54464336064, 0.0));
	expectedResults.add(Tuple2.of(0.0, 0.836329364957));
	expectedResults.add(Tuple2.of(0.607227075863, 0.268492484699));
	expectedResults.add(Tuple2.of(0.54464336064, 0.395445020996));
	expectedResults.add(Tuple2.of(0.0, 0.268492484699));
	expectedResults.add(Tuple2.of(0.194942293412, 0.0));

	for (Result<IntValue> result : hits.collect()) {
		int id = result.getVertexId0().getValue();
		assertEquals(expectedResults.get(id).f0, result.getHubScore().getValue(), ACCURACY);
		assertEquals(expectedResults.get(id).f1, result.getAuthorityScore().getValue(), ACCURACY);
	}
}
 

public static DataSet<IntValue> getIntDataSet(ExecutionEnvironment env) {
	List<IntValue> data = new ArrayList<>();

	data.add(new IntValue(1));
	data.add(new IntValue(2));
	data.add(new IntValue(2));
	data.add(new IntValue(3));
	data.add(new IntValue(3));
	data.add(new IntValue(3));
	data.add(new IntValue(4));
	data.add(new IntValue(4));
	data.add(new IntValue(4));
	data.add(new IntValue(4));
	data.add(new IntValue(5));
	data.add(new IntValue(5));
	data.add(new IntValue(5));
	data.add(new IntValue(5));
	data.add(new IntValue(5));

	Collections.shuffle(data);

	return env.fromCollection(data);
}
 

@Test
public void testReadTooShortInputLenient() throws IOException {
	try {
		final String fileContent = "666|777|888|999|555\n111|222|333|444\n666|777|888|999|555";
		final FileInputSplit split = createTempFile(fileContent);	
	
		final Configuration parameters = new Configuration();
		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, IntValue.class, IntValue.class, IntValue.class, IntValue.class);
		format.setLenient(true);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = createIntValues(5);
		
		assertNotNull(format.nextRecord(values));	// line okay
		assertNull(format.nextRecord(values));	// line too short
		assertNotNull(format.nextRecord(values));	// line okay
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 

@Test
public void testWithSimpleGraphWithMinimumScore() throws Exception {
	DataSet<Result<IntValue>> aa = undirectedSimpleGraph
		.run(new AdamicAdar<IntValue, NullValue, NullValue>()
			.setMinimumScore(0.75f));

	String expectedResult =
		"(0,1," + ilog[2] + ")\n" +
		"(0,2," + ilog[1] + ")\n" +
		"(0,3," + (ilog[1] + ilog[2]) + ")\n" +
		"(1,2," + (ilog[0] + ilog[3]) + ")\n" +
		"(1,3," + ilog[2] + ")\n" +
		"(2,3," + ilog[1] + ")";

	TestBaseUtils.compareResultAsText(aa.collect(), expectedResult);
}
 

@Override
public void reduce(Iterable<Tuple2<StringValue, IntValue>> values, Collector<Tuple2<StringValue, IntValue>> out) throws Exception {
	List<Tuple2<StringValue, IntValue>> all = new ArrayList<Tuple2<StringValue,IntValue>>();
	
	for (Tuple2<StringValue, IntValue> t : values) {
		all.add(t);
	}
	
	Tuple2<StringValue, IntValue> result = all.get(0);
	
	for (int i = 1; i < all.size(); i++) {
		Tuple2<StringValue, IntValue> e = all.get(i);
		result.f0.append(e.f0);
		result.f1.setValue(result.f1.getValue() + e.f1.getValue());
	}
	
	out.collect(result);
}
 

@Test
public void testWithSimpleGraph() throws Exception {
	DataSet<Result<IntValue>> pr = new PageRank<IntValue, NullValue, NullValue>(DAMPING_FACTOR, 20)
		.run(directedSimpleGraph);

	List<Double> expectedResults = new ArrayList<>();
	expectedResults.add(0.0909212166211);
	expectedResults.add(0.279516064311);
	expectedResults.add(0.129562719068);
	expectedResults.add(0.223268406353);
	expectedResults.add(0.185810377026);
	expectedResults.add(0.0909212166211);

	for (Result<IntValue> result : pr.collect()) {
		int id = result.getVertexId0().getValue();
		assertEquals(expectedResults.get(id), result.getPageRankScore().getValue(), ACCURACY);
	}
}
 

@Test
public void testWithUndirectedSimpleGraph() throws Exception {
	DataSet<Vertex<IntValue, LongValue>> outDegree = undirectedSimpleGraph
		.run(new VertexOutDegree<IntValue, NullValue, NullValue>()
			.setIncludeZeroDegreeVertices(true));

	String expectedResult =
		"(0,2)\n" +
		"(1,3)\n" +
		"(2,3)\n" +
		"(3,4)\n" +
		"(4,1)\n" +
		"(5,1)";

	TestBaseUtils.compareResultAsText(outDegree.collect(), expectedResult);
}
 

@Test
public void testReadTooShortInputLenient() throws IOException {
	try {
		final String fileContent = "666|777|888|999|555\n111|222|333|444\n666|777|888|999|555";
		final FileInputSplit split = createTempFile(fileContent);	
	
		final Configuration parameters = new Configuration();
		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, IntValue.class, IntValue.class, IntValue.class, IntValue.class);
		format.setLenient(true);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = createIntValues(5);
		
		assertNotNull(format.nextRecord(values));	// line okay
		assertNull(format.nextRecord(values));	// line too short
		assertNotNull(format.nextRecord(values));	// line okay
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 

private boolean verifyWrongPartitionHashKey(int position, int fieldNum) {
	final TypeComparator<Record> comparator = new RecordComparatorFactory(
		new int[] {position}, new Class[] {IntValue.class}).createComparator();
	final ChannelSelector<SerializationDelegate<Record>> selector = createChannelSelector(
		ShipStrategyType.PARTITION_HASH, comparator, 100);
	final SerializationDelegate<Record> delegate = new SerializationDelegate<>(new RecordSerializerFactory().getSerializer());

	Record record = new Record(2);
	record.setField(fieldNum, new IntValue(1));
	delegate.setInstance(record);

	try {
		selector.selectChannel(delegate);
	} catch (NullKeyFieldException re) {
		Assert.assertEquals(position, re.getFieldNumber());
		return true;
	}
	return false;
}
 

@Test
public void testNestedAggregateOfMutableValueTypes() throws Exception {
	/*
	 * Nested Aggregate of mutable value types
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<IntValue, LongValue, StringValue>> ds = ValueCollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple1<IntValue>> aggregateDs = ds.groupBy(1)
			.aggregate(Aggregations.MIN, 0)
			.aggregate(Aggregations.MIN, 0)
			.project(0);

	List<Tuple1<IntValue>> result = aggregateDs.collect();

	String expected = "1\n";

	compareResultAsTuples(result, expected);
}
 

@Override
public void invoke() throws Exception {
	RecordWriter<IntValue> writer = new RecordWriterBuilder<IntValue>().build(getEnvironment().getWriter(0));
	final int numberOfTimesToSend = getTaskConfiguration().getInteger(CONFIG_KEY, 0);

	final IntValue subtaskIndex = new IntValue(
			getEnvironment().getTaskInfo().getIndexOfThisSubtask());

	try {
		for (int i = 0; i < numberOfTimesToSend; i++) {
			writer.emit(subtaskIndex);
		}
		writer.flushAll();
	}
	finally {
		writer.clearBuffers();
	}
}
 

public static void prepareInputFile(MutableObjectIterator<Record> inIt, File inputFile, boolean insertInvalidData)
throws IOException {

	try (BufferedWriter bw = new BufferedWriter(new FileWriter(inputFile))) {
		if (insertInvalidData) {
			bw.write("####_I_AM_INVALID_########\n");
		}

		Record rec = new Record();
		while ((rec = inIt.next(rec)) != null) {
			IntValue key = rec.getField(0, IntValue.class);
			IntValue value = rec.getField(1, IntValue.class);

			bw.write(key.getValue() + "_" + value.getValue() + "\n");
		}
		if (insertInvalidData) {
			bw.write("####_I_AM_INVALID_########\n");
		}

		bw.flush();
	}
}
 

@Test
public void testRangePartitioningErased() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"1;2"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(3, LongValue.class, Order.DESCENDING);
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(6, ByteValue.class, Order.DESCENDING);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setRangePartitioned(o);

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNull(filtered);
}
 

@Override
public void invoke() throws Exception {
	RecordWriter<IntValue> writer = new RecordWriterBuilder().build(getEnvironment().getWriter(0));
	final int numberOfTimesToSend = getTaskConfiguration().getInteger(CONFIG_KEY, 0);

	final IntValue subtaskIndex = new IntValue(
			getEnvironment().getTaskInfo().getIndexOfThisSubtask());

	try {
		for (int i = 0; i < numberOfTimesToSend; i++) {
			writer.emit(subtaskIndex);
		}
		writer.flushAll();
	}
	finally {
		writer.clearBuffers();
	}
}
 

/**
 * Produce a {@code ValueArray} for the given {@code Value} type.
 *
 * @param cls {@code Value} class
 * @return {@code ValueArray} for given {@code Value} class
 */
@SuppressWarnings("unchecked")
public static <T> ValueArray<T> createValueArray(Class<? extends Value> cls) {
	if (ByteValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ByteValueArray();
	} else if (CharValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new CharValueArray();
	} else if (DoubleValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new DoubleValueArray();
	} else if (FloatValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new FloatValueArray();
	} else if (IntValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new IntValueArray();
	} else if (LongValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new LongValueArray();
	} else if (NullValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new NullValueArray();
	} else if (ShortValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ShortValueArray();
	} else if (StringValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new StringValueArray();
	} else {
		throw new IllegalArgumentException("Unable to create unbounded ValueArray for type " + cls);
	}
}
 

public static void prepareInputFile(MutableObjectIterator<Record> inIt, File inputFile, boolean insertInvalidData)
throws IOException {

	try (BufferedWriter bw = new BufferedWriter(new FileWriter(inputFile))) {
		if (insertInvalidData) {
			bw.write("####_I_AM_INVALID_########\n");
		}

		Record rec = new Record();
		while ((rec = inIt.next(rec)) != null) {
			IntValue key = rec.getField(0, IntValue.class);
			IntValue value = rec.getField(1, IntValue.class);

			bw.write(key.getValue() + "_" + value.getValue() + "\n");
		}
		if (insertInvalidData) {
			bw.write("####_I_AM_INVALID_########\n");
		}

		bw.flush();
	}
}
 

@Test
public void testOrderErased() {

	SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"1; 4"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(4, LongValue.class, Order.DESCENDING);
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(6, ByteValue.class, Order.DESCENDING);

	RequestedLocalProperties rlProp = new RequestedLocalProperties();
	rlProp.setOrdering(o);

	RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);

	assertNull(filtered);
}
 

@Test
public void testWithSimpleGraph() throws Exception {
	String expectedResult =
		"(0,1,((null),(3,0,3)))\n" +
		"(0,2,((null),(3,2,1)))\n" +
		"(2,1,((null),(3,0,3)))\n" +
		"(2,3,((null),(4,2,2)))\n" +
		"(3,1,((null),(3,0,3)))\n" +
		"(3,4,((null),(1,0,1)))\n" +
		"(5,3,((null),(4,2,2)))";

	DataSet<Edge<IntValue, Tuple2<NullValue, Degrees>>> targetDegrees = directedSimpleGraph
			.run(new EdgeTargetDegrees<>());

	TestBaseUtils.compareResultAsText(targetDegrees.collect(), expectedResult);
}
 

@Override
public void reduce(Iterable<Tuple3<T, T, FloatValue>> values, Collector<Tuple4<IntValue, T, T, FloatValue>> out)
		throws Exception {
	int groupCount = 0;
	int groupSpans = 1;

	groupSpansValue.setValue(groupSpans);

	for (Tuple3<T, T, FloatValue> edge : values) {
		output.f1 = edge.f0;
		output.f2 = edge.f1;
		output.f3 = edge.f2;

		out.collect(output);

		if (++groupCount == GROUP_SIZE) {
			groupCount = 0;
			groupSpansValue.setValue(++groupSpans);
		}
	}
}
 

@Test
public void testCreate() throws Exception {
	// supported primitive types
	Assert.assertEquals(StringSummaryAggregator.class, SummaryAggregatorFactory.create(String.class).getClass());
	Assert.assertEquals(ShortSummaryAggregator.class, SummaryAggregatorFactory.create(Short.class).getClass());
	Assert.assertEquals(IntegerSummaryAggregator.class, SummaryAggregatorFactory.create(Integer.class).getClass());
	Assert.assertEquals(LongSummaryAggregator.class, SummaryAggregatorFactory.create(Long.class).getClass());
	Assert.assertEquals(FloatSummaryAggregator.class, SummaryAggregatorFactory.create(Float.class).getClass());
	Assert.assertEquals(DoubleSummaryAggregator.class, SummaryAggregatorFactory.create(Double.class).getClass());
	Assert.assertEquals(BooleanSummaryAggregator.class, SummaryAggregatorFactory.create(Boolean.class).getClass());

	// supported value types
	Assert.assertEquals(ValueSummaryAggregator.StringValueSummaryAggregator.class, SummaryAggregatorFactory.create(StringValue.class).getClass());
	Assert.assertEquals(ValueSummaryAggregator.ShortValueSummaryAggregator.class, SummaryAggregatorFactory.create(ShortValue.class).getClass());
	Assert.assertEquals(ValueSummaryAggregator.IntegerValueSummaryAggregator.class, SummaryAggregatorFactory.create(IntValue.class).getClass());
	Assert.assertEquals(ValueSummaryAggregator.LongValueSummaryAggregator.class, SummaryAggregatorFactory.create(LongValue.class).getClass());
	Assert.assertEquals(ValueSummaryAggregator.FloatValueSummaryAggregator.class, SummaryAggregatorFactory.create(FloatValue.class).getClass());
	Assert.assertEquals(ValueSummaryAggregator.DoubleValueSummaryAggregator.class, SummaryAggregatorFactory.create(DoubleValue.class).getClass());
	Assert.assertEquals(ValueSummaryAggregator.BooleanValueSummaryAggregator.class, SummaryAggregatorFactory.create(BooleanValue.class).getClass());

	// some not well supported types - these fallback to ObjectSummaryAggregator
	Assert.assertEquals(ObjectSummaryAggregator.class, SummaryAggregatorFactory.create(Object.class).getClass());
	Assert.assertEquals(ObjectSummaryAggregator.class, SummaryAggregatorFactory.create(List.class).getClass());
}
 

@Test
public void testSimpleGraph() throws Exception {
	String expectedResult =
		"(0,2,1)\n" +
		"(1,3,2)\n" +
		"(2,3,2)\n" +
		"(3,4,1)\n" +
		"(4,1,0)\n" +
		"(5,1,0)";

	DataSet<Result<IntValue>> cc = directedSimpleGraph
		.run(new LocalClusteringCoefficient<>());

	TestBaseUtils.compareResultAsText(cc.collect(), expectedResult);
}
 

@Test
public void testSimpleGraphSorted() throws Exception {
	DataSet<Result<IntValue>> tl = directedSimpleGraph
		.run(new TriangleListing<IntValue, NullValue, NullValue>()
			.setSortTriangleVertices(true));

	String expectedResult =
		"(0,1,2,41)\n" +
		"(1,2,3,22)";

	TestBaseUtils.compareResultAsText(tl.collect(), expectedResult);
}
 

@Test
public void testWithSimpleGraphWithMinimumScore() throws Exception {
	DataSet<Result<IntValue>> ji = undirectedSimpleGraph
		.run(new JaccardIndex<IntValue, NullValue, NullValue>()
			.setMinimumScore(1, 2));

	String expectedResult =
		"(0,3,2,4)\n" +
		"(1,2,2,4)\n" +
		"(4,5,1,1)\n";

	TestBaseUtils.compareResultAsText(ji.collect(), expectedResult);
}
 

@Test
public void testMultiKeys() {
	final int numberOfChannels = 100;
	final int numRecords = 5000;
	final TypeComparator<Record> multiComp = new RecordComparatorFactory(
		new int[] {0,1, 3}, new Class[] {IntValue.class, StringValue.class, DoubleValue.class}).createComparator();

	final ChannelSelector<SerializationDelegate<Record>> selector = createChannelSelector(
		ShipStrategyType.PARTITION_HASH, multiComp, numberOfChannels);
	final SerializationDelegate<Record> delegate = new SerializationDelegate<>(new RecordSerializerFactory().getSerializer());

	int[] hits = new int[numberOfChannels];
	for (int i = 0; i < numRecords; i++) {
		Record record = new Record(4);
		record.setField(0, new IntValue(i));
		record.setField(1, new StringValue("AB" + i + "CD" + i));
		record.setField(3, new DoubleValue(i * 3.141d));
		delegate.setInstance(record);

		int channel = selector.selectChannel(delegate);
		hits[channel]++;
	}

	int totalHitCount = 0;
	for (int hit : hits) {
		assertTrue(hit > 0);
		totalHitCount += hit;
	}
	assertTrue(totalHitCount == numRecords);
}
 

@Override
public IntValue[] getValidTestResults() {
	return new IntValue[] {
		new IntValue(0), new IntValue(1), new IntValue(576), new IntValue(-877678),
		new IntValue(Integer.MAX_VALUE), new IntValue(Integer.MIN_VALUE), new IntValue(1239)
	};
}
 

@Test
public void testSparseParse() {
	try {
		final String fileContent =
				"111|222|333|444|555|666|777|888|999|000|\n"+
				"000|999|888|777|666|555|444|333|222|111|";
		final FileInputSplit split = createTempFile(fileContent);	
	
		final Configuration parameters = new Configuration();
		
		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, null, null, IntValue.class, null, null, null, IntValue.class);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = createIntValues(3);
		
		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(111, ((IntValue) values[0]).getValue());
		assertEquals(444, ((IntValue) values[1]).getValue());
		assertEquals(888, ((IntValue) values[2]).getValue());
		
		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(000, ((IntValue) values[0]).getValue());
		assertEquals(777, ((IntValue) values[1]).getValue());
		assertEquals(333, ((IntValue) values[2]).getValue());
		
		assertNull(format.nextRecord(values));
		assertTrue(format.reachedEnd());
	}
	catch (Exception ex) {
		System.err.println(ex.getMessage());
		ex.printStackTrace();
		fail("Test erroneous");
	}
}
 

private Value[] createIntValues(int num) {
	Value[] v = new Value[num];
	
	for (int i = 0; i < num; i++) {
		v[i] = new IntValue();
	}
	
	return v;
}
 

@Test
public void testWithSimpleGraph() throws Exception {
	String expectedResult =
		"(0,1,((null),2,3))\n" +
		"(0,2,((null),2,3))\n" +
		"(1,0,((null),3,2))\n" +
		"(1,2,((null),3,3))\n" +
		"(1,3,((null),3,4))\n" +
		"(2,0,((null),3,2))\n" +
		"(2,1,((null),3,3))\n" +
		"(2,3,((null),3,4))\n" +
		"(3,1,((null),4,3))\n" +
		"(3,2,((null),4,3))\n" +
		"(3,4,((null),4,1))\n" +
		"(3,5,((null),4,1))\n" +
		"(4,3,((null),1,4))\n" +
		"(5,3,((null),1,4))";

	DataSet<Edge<IntValue, Tuple3<NullValue, LongValue, LongValue>>> degreePairOnSourceId = undirectedSimpleGraph
		.run(new EdgeDegreePair<IntValue, NullValue, NullValue>()
			.setReduceOnTargetId(false));

	TestBaseUtils.compareResultAsText(degreePairOnSourceId.collect(), expectedResult);

	DataSet<Edge<IntValue, Tuple3<NullValue, LongValue, LongValue>>> degreePairOnTargetId = undirectedSimpleGraph
		.run(new EdgeDegreePair<IntValue, NullValue, NullValue>()
			.setReduceOnTargetId(true));

	TestBaseUtils.compareResultAsText(degreePairOnTargetId.collect(), expectedResult);
}
 

@Test
public void testSimpleGraph() throws Exception {
	String expectedResult =
		"(0,2,1)\n" +
		"(1,3,2)\n" +
		"(2,3,2)\n" +
		"(3,4,1)\n" +
		"(4,1,0)\n" +
		"(5,1,0)";

	DataSet<Result<IntValue>> cc = undirectedSimpleGraph
		.run(new LocalClusteringCoefficient<>());

	TestBaseUtils.compareResultAsText(cc.collect(), expectedResult);
}