Java Code Examples for org.apache.flink.api.java.functions.SemanticPropUtil#getSemanticPropsSingleFromString()

@Test
public void testUniqueFieldsPreserved2() {

	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;1;2;3;4"}, null, null, tupleInfo, tupleInfo);

	LocalProperties lProps = LocalProperties.forGrouping(new FieldList(1,2));
	lProps = lProps.addUniqueFields(new FieldSet(0,1,2));
	lProps = lProps.addUniqueFields(new FieldSet(3,4));
	lProps = lProps.addUniqueFields(new FieldSet(4,5,6));

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldSet expected1 = new FieldSet(0,1,2);
	FieldSet expected2 = new FieldSet(3,4);

	assertNull(filtered.getOrdering());
	assertNotNull(filtered.getGroupedFields());
	assertEquals(2, filtered.getGroupedFields().size());
	assertTrue(filtered.getGroupedFields().contains(1));
	assertTrue(filtered.getGroupedFields().contains(2));
	assertNotNull(filtered.getUniqueFields());
	assertEquals(2, filtered.getUniqueFields().size());
	assertTrue(filtered.getUniqueFields().contains(expected1));
	assertTrue(filtered.getUniqueFields().contains(expected2));
}
 

@Test
public void testGroupingPreserved2() {

	SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"3->0;5->2;1->3"}, null, null, tupleInfo, tupleInfo);

	RequestedLocalProperties rlProp = new RequestedLocalProperties();
	rlProp.setGroupedFields(new FieldSet(0, 2, 3));

	RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);

	assertNotNull(filtered);
	assertNotNull(filtered.getGroupedFields());
	assertEquals(3, filtered.getGroupedFields().size());
	assertTrue(filtered.getGroupedFields().contains(3));
	assertTrue(filtered.getGroupedFields().contains(5));
	assertTrue(filtered.getGroupedFields().contains(1));
	assertNull(filtered.getOrdering());
}
 

@Test
public void testHashPartitioningPreserved1() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"0;3;4"}, null, null, tupleInfo, tupleInfo);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setHashPartitioned(new FieldSet(0, 3, 4));

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNotNull(filtered);
	assertEquals(PartitioningProperty.HASH_PARTITIONED, filtered.getPartitioning());
	assertNotNull(filtered.getPartitionedFields());
	assertEquals(3, filtered.getPartitionedFields().size());
	assertTrue(filtered.getPartitionedFields().contains(0));
	assertTrue(filtered.getPartitionedFields().contains(3));
	assertTrue(filtered.getPartitionedFields().contains(4));
	assertNull(filtered.getDataDistribution());
	assertNull(filtered.getCustomPartitioner());
	assertNull(filtered.getOrdering());
}
 

@Test
public void testSortingPreserved2() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->3;2->7;5->1"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(3, gFields.size());
	assertTrue(gFields.contains(3));
	assertTrue(gFields.contains(7));
	assertTrue(gFields.contains(1));
	assertNotNull(order);
	assertEquals(3, order.getNumberOfFields());
	assertEquals(7, order.getFieldNumber(0).intValue());
	assertEquals(3, order.getFieldNumber(1).intValue());
	assertEquals(1, order.getFieldNumber(2).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(Order.DESCENDING, order.getOrder(1));
	assertEquals(Order.DESCENDING, order.getOrder(2));
	assertEquals(IntValue.class, order.getType(0));
	assertEquals(StringValue.class, order.getType(1));
	assertEquals(LongValue.class, order.getType(2));
	assertNull(filtered.getUniqueFields());
}
 

@Test
public void testSortingPreserved2() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->3;2->7;5->1"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(3, gFields.size());
	assertTrue(gFields.contains(3));
	assertTrue(gFields.contains(7));
	assertTrue(gFields.contains(1));
	assertNotNull(order);
	assertEquals(3, order.getNumberOfFields());
	assertEquals(7, order.getFieldNumber(0).intValue());
	assertEquals(3, order.getFieldNumber(1).intValue());
	assertEquals(1, order.getFieldNumber(2).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(Order.DESCENDING, order.getOrder(1));
	assertEquals(Order.DESCENDING, order.getOrder(2));
	assertEquals(IntValue.class, order.getType(0));
	assertEquals(StringValue.class, order.getType(1));
	assertEquals(LongValue.class, order.getType(2));
	assertNull(filtered.getUniqueFields());
}
 

@Test(expected = IndexOutOfBoundsException.class)
public void testInvalidInputIndex() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	gprops.setHashPartitioned(new FieldList(0, 1));

	gprops.filterBySemanticProperties(sprops, 1);
}
 

@Test
public void testRangePartitioningPreserved2() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"7->3;1->1;2->6"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(3, LongValue.class, Order.DESCENDING);
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(6, ByteValue.class, Order.DESCENDING);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setRangePartitioned(o);

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNotNull(filtered);
	assertEquals(PartitioningProperty.RANGE_PARTITIONED, filtered.getPartitioning());
	assertNotNull(filtered.getOrdering());
	assertEquals(3, filtered.getOrdering().getNumberOfFields());
	assertEquals(7, filtered.getOrdering().getFieldNumber(0).intValue());
	assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
	assertEquals(2, filtered.getOrdering().getFieldNumber(2).intValue());
	assertEquals(LongValue.class, filtered.getOrdering().getType(0));
	assertEquals(IntValue.class, filtered.getOrdering().getType(1));
	assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
	assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
	assertNull(filtered.getPartitionedFields());
	assertNull(filtered.getDataDistribution());
	assertNull(filtered.getCustomPartitioner());
}
 

@Test
public void testRangePartitioningPreserved1() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"1;3;6"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(3, LongValue.class, Order.DESCENDING);
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(6, ByteValue.class, Order.DESCENDING);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setRangePartitioned(o);

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNotNull(filtered);
	assertEquals(PartitioningProperty.RANGE_PARTITIONED, filtered.getPartitioning());
	assertNotNull(filtered.getOrdering());
	assertEquals(3, filtered.getOrdering().getNumberOfFields());
	assertEquals(3, filtered.getOrdering().getFieldNumber(0).intValue());
	assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
	assertEquals(6, filtered.getOrdering().getFieldNumber(2).intValue());
	assertEquals(LongValue.class, filtered.getOrdering().getType(0));
	assertEquals(IntValue.class, filtered.getOrdering().getType(1));
	assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
	assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
	assertNull(filtered.getPartitionedFields());
	assertNull(filtered.getDataDistribution());
	assertNull(filtered.getCustomPartitioner());
}
 

@Test
public void testRangePartitioningPreserved3() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"7->3;1->1;2->6"}, null, null, tupleInfo, tupleInfo);

	DataDistribution dd = new MockDistribution();
	Ordering o = new Ordering();
	o.appendOrdering(3, LongValue.class, Order.DESCENDING);
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(6, ByteValue.class, Order.DESCENDING);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setRangePartitioned(o, dd);

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNotNull(filtered);
	assertEquals(PartitioningProperty.RANGE_PARTITIONED, filtered.getPartitioning());
	assertNotNull(filtered.getOrdering());
	assertEquals(3, filtered.getOrdering().getNumberOfFields());
	assertEquals(7, filtered.getOrdering().getFieldNumber(0).intValue());
	assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
	assertEquals(2, filtered.getOrdering().getFieldNumber(2).intValue());
	assertEquals(LongValue.class, filtered.getOrdering().getType(0));
	assertEquals(IntValue.class, filtered.getOrdering().getType(1));
	assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
	assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
	assertNotNull(filtered.getDataDistribution());
	assertEquals(dd, filtered.getDataDistribution());
	assertNull(filtered.getPartitionedFields());
	assertNull(filtered.getCustomPartitioner());
}
 

@Test
public void testRangePartitioningPreserved1() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"1;3;6"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(3, LongValue.class, Order.DESCENDING);
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(6, ByteValue.class, Order.DESCENDING);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setRangePartitioned(o);

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNotNull(filtered);
	assertEquals(PartitioningProperty.RANGE_PARTITIONED, filtered.getPartitioning());
	assertNotNull(filtered.getOrdering());
	assertEquals(3, filtered.getOrdering().getNumberOfFields());
	assertEquals(3, filtered.getOrdering().getFieldNumber(0).intValue());
	assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
	assertEquals(6, filtered.getOrdering().getFieldNumber(2).intValue());
	assertEquals(LongValue.class, filtered.getOrdering().getType(0));
	assertEquals(IntValue.class, filtered.getOrdering().getType(1));
	assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
	assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
	assertNull(filtered.getPartitionedFields());
	assertNull(filtered.getDataDistribution());
	assertNull(filtered.getCustomPartitioner());
}
 

@Test
public void testGroupingErased() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->4;2->0"}, null, null, tupleInfo, tupleInfo);

	LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 2, 3));

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);

	assertNull(filtered.getGroupedFields());
	assertNull(filtered.getOrdering());
	assertNull(filtered.getUniqueFields());
}
 

@Test
public void testSortingPreserved1() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;2;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(3, gFields.size());
	assertTrue(gFields.contains(0));
	assertTrue(gFields.contains(2));
	assertTrue(gFields.contains(5));
	assertNotNull(order);
	assertEquals(3, order.getNumberOfFields());
	assertEquals(2, order.getFieldNumber(0).intValue());
	assertEquals(0, order.getFieldNumber(1).intValue());
	assertEquals(5, order.getFieldNumber(2).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(Order.DESCENDING, order.getOrder(1));
	assertEquals(Order.DESCENDING, order.getOrder(2));
	assertEquals(IntValue.class, order.getType(0));
	assertEquals(StringValue.class, order.getType(1));
	assertEquals(LongValue.class, order.getType(2));
	assertNull(filtered.getUniqueFields());
}
 

@Test
public void testAnyPartitioningErased() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"1;2"}, null, null, tupleInfo, tupleInfo);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setAnyPartitioning(new FieldSet(0, 3, 4));

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNull(filtered);
}
 

@Test
public void testEraseAll2() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"3;4"}, null, null, tupleInfo, tupleInfo);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setAnyPartitioning(new FieldSet(0, 1, 2));

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNull(filtered);
}
 

@Test
public void testHashPartitioningErased() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"1;2"}, null, null, tupleInfo, tupleInfo);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setHashPartitioned(new FieldSet(0, 3, 4));

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNull(filtered);
}
 

@Test(expected = IndexOutOfBoundsException.class)
public void testInvalidInputIndex() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo);

	LocalProperties lprops = LocalProperties.forGrouping(new FieldList(0,1));

	lprops.filterBySemanticProperties(sprops, 1);
}
 

@Test
public void testRandomDistributionErased() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"0;1;2"}, null, null, tupleInfo, tupleInfo);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setRandomPartitioning();

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNull(filtered);
}
 

@Test
public void testReplicationErased() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"0;1;2"}, null, null, tupleInfo, tupleInfo);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setFullyReplicated();

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNull(filtered);
}
 

@Test
public void testReplicationErased() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"0;1;2"}, null, null, tupleInfo, tupleInfo);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setFullyReplicated();

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNull(filtered);
}
 

/**
 * Adds semantic information about forwarded fields of the user-defined function.
 * The forwarded fields information declares fields which are never modified by the function and
 * which are forwarded at the same position to the output or unchanged copied to another position in the output.
 *
 * <p>Fields that are forwarded at the same position are specified by their position.
 * The specified position must be valid for the input and output data type and have the same type.
 * For example <code>withForwardedFields("f2")</code> declares that the third field of a Java input tuple is
 * copied to the third field of an output tuple.
 *
 * <p>Fields which are unchanged copied to another position in the output are declared by specifying the
 * source field reference in the input and the target field reference in the output.
 * {@code withForwardedFields("f0->f2")} denotes that the first field of the Java input tuple is
 * unchanged copied to the third field of the Java output tuple. When using a wildcard ("*") ensure that
 * the number of declared fields and their types in input and output type match.
 *
 * <p>Multiple forwarded fields can be annotated in one ({@code withForwardedFields("f2; f3->f0; f4")})
 * or separate Strings ({@code withForwardedFields("f2", "f3->f0", "f4")}).
 * Please refer to the JavaDoc of {@link org.apache.flink.api.common.functions.Function} or Flink's documentation for
 * details on field references such as nested fields and wildcard.
 *
 * <p>It is not possible to override existing semantic information about forwarded fields which was
 * for example added by a {@link org.apache.flink.api.java.functions.FunctionAnnotation.ForwardedFields} class annotation.
 *
 * <p><b>NOTE: Adding semantic information for functions is optional!
 * If used correctly, semantic information can help the Flink optimizer to generate more efficient execution plans.
 * However, incorrect semantic information can cause the optimizer to generate incorrect execution plans which compute wrong results!
 * So be careful when adding semantic information.
 * </b>
 *
 * @param forwardedFields A list of field forward expressions.
 * @return This operator with annotated forwarded field information.
 *
 * @see org.apache.flink.api.java.functions.FunctionAnnotation
 * @see org.apache.flink.api.java.functions.FunctionAnnotation.ForwardedFields
 */
public O withForwardedFields(String... forwardedFields) {

	if (this.udfSemantics == null) {
		// extract semantic properties from function annotations
		setSemanticProperties(extractSemanticAnnotations(getFunction().getClass()));
	}

	if (this.udfSemantics == null
			|| this.analyzedUdfSemantics) { // discard analyzed semantic properties
		setSemanticProperties(new SingleInputSemanticProperties());
		SemanticPropUtil.getSemanticPropsSingleFromString(this.udfSemantics, forwardedFields, null, null, this.getInputType(), this.getResultType());
	} else {
		if (udfWithForwardedFieldsAnnotation(getFunction().getClass())) {
			// refuse semantic information as it would override the function annotation
			throw new SemanticProperties.InvalidSemanticAnnotationException("Forwarded field information " +
					"has already been added by a function annotation for this operator. " +
					"Cannot overwrite function annotations.");
		} else {
			SemanticPropUtil.getSemanticPropsSingleFromString(this.udfSemantics, forwardedFields, null, null, this.getInputType(), this.getResultType());
		}
	}

	@SuppressWarnings("unchecked")
	O returnType = (O) this;
	return returnType;
}