org.apache.flink.api.common.operators.util.FieldList Java Examples
The following examples show how to use
org.apache.flink.api.common.operators.util.FieldList.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LocalPropertiesFilteringTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testSortingErased() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;5"}, null, null, tupleInfo, tupleInfo); Ordering o = new Ordering(); o.appendOrdering(2, IntValue.class, Order.ASCENDING); o.appendOrdering(0, StringValue.class, Order.DESCENDING); o.appendOrdering(5, LongValue.class, Order.DESCENDING); LocalProperties lProps = LocalProperties.forOrdering(o); LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0); FieldList gFields = filtered.getGroupedFields(); Ordering order = filtered.getOrdering(); assertNull(gFields); assertNull(order); assertNull(filtered.getUniqueFields()); }
Example #2
Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testCustomPartitioningPreserved1() { SingleInputSemanticProperties sprops = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo); GlobalProperties gprops = new GlobalProperties(); Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner(); gprops.setCustomPartitioned(new FieldList(0, 4), myP); GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0); assertEquals(PartitioningProperty.CUSTOM_PARTITIONING, result.getPartitioning()); FieldList pFields = result.getPartitioningFields(); assertEquals(2, pFields.size()); assertTrue(pFields.contains(0)); assertTrue(pFields.contains(4)); assertEquals(myP, result.getCustomPartitioner()); }
Example #3
Source File: LocalPropertiesFilteringTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testGroupingPreserved1() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;2;3"}, null, null, tupleInfo, tupleInfo); LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 2, 3)); LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0); assertNotNull(filtered.getGroupedFields()); assertEquals(3, filtered.getGroupedFields().size()); assertTrue(filtered.getGroupedFields().contains(0)); assertTrue(filtered.getGroupedFields().contains(2)); assertTrue(filtered.getGroupedFields().contains(3)); assertNull(filtered.getOrdering()); assertNull(filtered.getUniqueFields()); }
Example #4
Source File: GlobalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testCustomPartitioningErased() { SingleInputSemanticProperties sprops = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo); GlobalProperties gprops = new GlobalProperties(); Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner(); gprops.setCustomPartitioned(new FieldList(0, 4), myP); GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0); assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning()); assertNull(result.getPartitioningFields()); assertNull(result.getCustomPartitioner()); }
Example #5
Source File: WorksetIterationNode.java From flink with Apache License 2.0 | 6 votes |
/** * Creates a new node with a single input for the optimizer plan. * * @param iteration The iteration operator that the node represents. */ public WorksetIterationNode(DeltaIterationBase<?, ?> iteration) { super(iteration); final int[] ssKeys = iteration.getSolutionSetKeyFields(); if (ssKeys == null || ssKeys.length == 0) { throw new CompilerException("Invalid WorksetIteration: No key fields defined for the solution set."); } this.solutionSetKeyFields = new FieldList(ssKeys); this.partitionedProperties = new GlobalProperties(); this.partitionedProperties.setHashPartitioned(this.solutionSetKeyFields); int weight = iteration.getMaximumNumberOfIterations() > 0 ? iteration.getMaximumNumberOfIterations() : DEFAULT_COST_WEIGHT; if (weight > OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT) { weight = OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT; } this.costWeight = weight; this.dataProperties = Collections.<OperatorDescriptorDual>singletonList(new WorksetOpDescriptor(this.solutionSetKeyFields)); }
Example #6
Source File: LocalPropertiesFilteringTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testAllErased1() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, null, tupleInfo, tupleInfo); LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 1, 2)); lProps = lProps.addUniqueFields(new FieldSet(3,4)); lProps = lProps.addUniqueFields(new FieldSet(5,6)); LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0); assertNull(filtered.getGroupedFields()); assertNull(filtered.getOrdering()); assertNull(filtered.getUniqueFields()); }
Example #7
Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testHashPartitioningPreserved1() { SingleInputSemanticProperties sprops = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo); GlobalProperties gprops = new GlobalProperties(); gprops.setHashPartitioned(new FieldList(0, 1, 4)); GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0); assertEquals(PartitioningProperty.HASH_PARTITIONED, result.getPartitioning()); FieldList pFields = result.getPartitioningFields(); assertEquals(3, pFields.size()); assertTrue(pFields.contains(0)); assertTrue(pFields.contains(1)); assertTrue(pFields.contains(4)); }
Example #8
Source File: GlobalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testAllErased2() { SingleInputSemanticProperties semProps = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(semProps, new String[]{"2"}, null, null, tupleInfo, tupleInfo); GlobalProperties gprops = new GlobalProperties(); gprops.setHashPartitioned(new FieldList(0, 1)); gprops.addUniqueFieldCombination(new FieldSet(3, 4)); gprops.addUniqueFieldCombination(new FieldSet(5, 6)); GlobalProperties result = gprops.filterBySemanticProperties(semProps, 0); assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning()); assertNull(result.getPartitioningFields()); assertNull(result.getPartitioningOrdering()); assertNull(result.getUniqueFieldCombination()); }
Example #9
Source File: Utils.java From flink with Apache License 2.0 | 6 votes |
private static <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder, ExecutionConfig executionConfig) { TypeComparator<T> comparator; if (typeInfo instanceof CompositeType) { comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig); } else if (typeInfo instanceof AtomicType) { // handle grouping of atomic types comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig); } else { throw new RuntimeException("Unrecognized type: " + typeInfo); } return new RuntimeComparatorFactory<>(comparator); }
Example #10
Source File: GlobalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testCustomPartitioningPreserved2() { SingleInputSemanticProperties sprops = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0->1; 1->2; 4->3"}, null, null, tupleInfo, tupleInfo); GlobalProperties gprops = new GlobalProperties(); Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner(); gprops.setCustomPartitioned(new FieldList(0, 4), myP); GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0); assertEquals(PartitioningProperty.CUSTOM_PARTITIONING, result.getPartitioning()); FieldList pFields = result.getPartitioningFields(); assertEquals(2, pFields.size()); assertTrue(pFields.contains(1)); assertTrue(pFields.contains(3)); assertEquals(myP, result.getCustomPartitioner()); }
Example #11
Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testAllErased1() { SingleInputSemanticProperties semProps = new SingleInputSemanticProperties(); GlobalProperties gprops = new GlobalProperties(); gprops.setHashPartitioned(new FieldList(0, 1)); gprops.addUniqueFieldCombination(new FieldSet(3, 4)); gprops.addUniqueFieldCombination(new FieldSet(5, 6)); GlobalProperties result = gprops.filterBySemanticProperties(semProps, 0); assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning()); assertNull(result.getPartitioningFields()); assertNull(result.getPartitioningOrdering()); assertNull(result.getUniqueFieldCombination()); }
Example #12
Source File: JavaApiPostPass.java From flink with Apache License 2.0 | 6 votes |
@SuppressWarnings("unchecked") private <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder) { TypeComparator<T> comparator; if (typeInfo instanceof CompositeType) { comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig); } else if (typeInfo instanceof AtomicType) { // handle grouping of atomic types comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig); } else { throw new RuntimeException("Unrecognized type: " + typeInfo); } return new RuntimeComparatorFactory<T>(comparator); }
Example #13
Source File: LocalPropertiesFilteringTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testSortingPreserved4() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"2->7;5"}, null, null, tupleInfo, tupleInfo); Ordering o = new Ordering(); o.appendOrdering(2, IntValue.class, Order.ASCENDING); o.appendOrdering(0, StringValue.class, Order.DESCENDING); o.appendOrdering(5, LongValue.class, Order.DESCENDING); LocalProperties lProps = LocalProperties.forOrdering(o); LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0); FieldList gFields = filtered.getGroupedFields(); Ordering order = filtered.getOrdering(); assertNotNull(gFields); assertEquals(1, gFields.size()); assertTrue(gFields.contains(7)); assertNotNull(order); assertEquals(1, order.getNumberOfFields()); assertEquals(7, order.getFieldNumber(0).intValue()); assertEquals(Order.ASCENDING, order.getOrder(0)); assertEquals(IntValue.class, order.getType(0)); assertNull(filtered.getUniqueFields()); }
Example #14
Source File: LocalPropertiesFilteringTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testSortingPreserved1() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;2;5"}, null, null, tupleInfo, tupleInfo); Ordering o = new Ordering(); o.appendOrdering(2, IntValue.class, Order.ASCENDING); o.appendOrdering(0, StringValue.class, Order.DESCENDING); o.appendOrdering(5, LongValue.class, Order.DESCENDING); LocalProperties lProps = LocalProperties.forOrdering(o); LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0); FieldList gFields = filtered.getGroupedFields(); Ordering order = filtered.getOrdering(); assertNotNull(gFields); assertEquals(3, gFields.size()); assertTrue(gFields.contains(0)); assertTrue(gFields.contains(2)); assertTrue(gFields.contains(5)); assertNotNull(order); assertEquals(3, order.getNumberOfFields()); assertEquals(2, order.getFieldNumber(0).intValue()); assertEquals(0, order.getFieldNumber(1).intValue()); assertEquals(5, order.getFieldNumber(2).intValue()); assertEquals(Order.ASCENDING, order.getOrder(0)); assertEquals(Order.DESCENDING, order.getOrder(1)); assertEquals(Order.DESCENDING, order.getOrder(2)); assertEquals(IntValue.class, order.getType(0)); assertEquals(StringValue.class, order.getType(1)); assertEquals(LongValue.class, order.getType(2)); assertNull(filtered.getUniqueFields()); }
Example #15
Source File: LocalPropertiesFilteringTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testAllErased2() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"5"}, null, null, tupleInfo, tupleInfo); LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 1, 2)); lProps = lProps.addUniqueFields(new FieldSet(3,4)); LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0); assertNull(filtered.getGroupedFields()); assertNull(filtered.getOrdering()); assertNull(filtered.getUniqueFields()); }
Example #16
Source File: LocalPropertiesFilteringTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testSortingPreserved1() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;2;5"}, null, null, tupleInfo, tupleInfo); Ordering o = new Ordering(); o.appendOrdering(2, IntValue.class, Order.ASCENDING); o.appendOrdering(0, StringValue.class, Order.DESCENDING); o.appendOrdering(5, LongValue.class, Order.DESCENDING); LocalProperties lProps = LocalProperties.forOrdering(o); LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0); FieldList gFields = filtered.getGroupedFields(); Ordering order = filtered.getOrdering(); assertNotNull(gFields); assertEquals(3, gFields.size()); assertTrue(gFields.contains(0)); assertTrue(gFields.contains(2)); assertTrue(gFields.contains(5)); assertNotNull(order); assertEquals(3, order.getNumberOfFields()); assertEquals(2, order.getFieldNumber(0).intValue()); assertEquals(0, order.getFieldNumber(1).intValue()); assertEquals(5, order.getFieldNumber(2).intValue()); assertEquals(Order.ASCENDING, order.getOrder(0)); assertEquals(Order.DESCENDING, order.getOrder(1)); assertEquals(Order.DESCENDING, order.getOrder(2)); assertEquals(IntValue.class, order.getType(0)); assertEquals(StringValue.class, order.getType(1)); assertEquals(LongValue.class, order.getType(2)); assertNull(filtered.getUniqueFields()); }
Example #17
Source File: JavaApiPostPass.java From flink with Apache License 2.0 | 5 votes |
private static final boolean[] getSortOrders(FieldList keys, boolean[] orders) { if (orders == null) { orders = new boolean[keys.size()]; Arrays.fill(orders, true); } return orders; }
Example #18
Source File: Channel.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public void setShipStrategy(ShipStrategyType strategy, FieldList keys, boolean[] sortDirection, Partitioner<?> partitioner, DataExchangeMode dataExchangeMode) { this.shipStrategy = strategy; this.shipKeys = keys; this.shipSortOrder = sortDirection; this.partitioner = partitioner; this.dataExchangeMode = dataExchangeMode; this.globalProps = null; // reset the global properties }
Example #19
Source File: SingleInputPlanNode.java From flink with Apache License 2.0 | 5 votes |
public SingleInputPlanNode(OptimizerNode template, String nodeName, Channel input, DriverStrategy driverStrategy, FieldList driverKeyFields, boolean[] driverSortOrders) { super(template, nodeName, driverStrategy); this.input = input; this.comparators = new TypeComparatorFactory<?>[driverStrategy.getNumRequiredComparators()]; this.driverKeys = new FieldList[driverStrategy.getNumRequiredComparators()]; this.driverSortOrders = new boolean[driverStrategy.getNumRequiredComparators()][]; if(driverStrategy.getNumRequiredComparators() > 0) { this.driverKeys[0] = driverKeyFields; this.driverSortOrders[0] = driverSortOrders; } if (this.input.getShipStrategy() == ShipStrategyType.BROADCAST) { this.input.setReplicationFactor(getParallelism()); } final PlanNode predNode = input.getSource(); if (predNode.branchPlan != null && !predNode.branchPlan.isEmpty()) { if (this.branchPlan == null) { this.branchPlan = new HashMap<OptimizerNode, PlanNode>(); } this.branchPlan.putAll(predNode.branchPlan); } }
Example #20
Source File: SingleInputPlanNode.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Sets the key field information for the specified driver comparator. * * @param keys The key field indexes for the specified driver comparator. * @param sortOrder The key sort order for the specified driver comparator. * @param id The ID of the driver comparator. */ public void setDriverKeyInfo(FieldList keys, boolean[] sortOrder, int id) { if(id < 0 || id >= driverKeys.length) { throw new CompilerException("Invalid id for driver key information. DriverStrategy requires only " +super.getDriverStrategy().getNumRequiredComparators()+" comparators."); } this.driverKeys[id] = keys; this.driverSortOrders[id] = sortOrder; }
Example #21
Source File: FieldListTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private static void check(FieldList set, int... elements) { if (elements == null) { assertEquals(0, set.size()); return; } assertEquals(elements.length, set.size()); // test contains for (int i : elements) { set.contains(i); } // test to array { int[] arr = set.toArray(); assertTrue(Arrays.equals(arr, elements)); } { int[] fromIter = new int[set.size()]; Iterator<Integer> iter = set.iterator(); for (int i = 0; i < fromIter.length; i++) { fromIter[i] = iter.next(); } assertFalse(iter.hasNext()); assertTrue(Arrays.equals(fromIter, elements)); } }
Example #22
Source File: Utils.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public static FieldList createOrderedFromSet(FieldSet set) { if (set instanceof FieldList) { return (FieldList) set; } else { final int[] cols = set.toArray(); Arrays.sort(cols); return new FieldList(cols); } }
Example #23
Source File: LocalPropertiesFilteringTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testSortingPreserved2() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->3;2->7;5->1"}, null, null, tupleInfo, tupleInfo); Ordering o = new Ordering(); o.appendOrdering(2, IntValue.class, Order.ASCENDING); o.appendOrdering(0, StringValue.class, Order.DESCENDING); o.appendOrdering(5, LongValue.class, Order.DESCENDING); LocalProperties lProps = LocalProperties.forOrdering(o); LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0); FieldList gFields = filtered.getGroupedFields(); Ordering order = filtered.getOrdering(); assertNotNull(gFields); assertEquals(3, gFields.size()); assertTrue(gFields.contains(3)); assertTrue(gFields.contains(7)); assertTrue(gFields.contains(1)); assertNotNull(order); assertEquals(3, order.getNumberOfFields()); assertEquals(7, order.getFieldNumber(0).intValue()); assertEquals(3, order.getFieldNumber(1).intValue()); assertEquals(1, order.getFieldNumber(2).intValue()); assertEquals(Order.ASCENDING, order.getOrder(0)); assertEquals(Order.DESCENDING, order.getOrder(1)); assertEquals(Order.DESCENDING, order.getOrder(2)); assertEquals(IntValue.class, order.getType(0)); assertEquals(StringValue.class, order.getType(1)); assertEquals(LongValue.class, order.getType(2)); assertNull(filtered.getUniqueFields()); }
Example #24
Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testHashPartitioningErased() { SingleInputSemanticProperties sprops = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo); GlobalProperties gprops = new GlobalProperties(); gprops.setHashPartitioned(new FieldList(0, 1, 4)); GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0); assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning()); assertNull(result.getPartitioningFields()); }
Example #25
Source File: Utils.java From flink with Apache License 2.0 | 5 votes |
private static boolean[] getSortOrders(FieldList keys, boolean[] orders) { if (orders == null) { orders = new boolean[keys.size()]; Arrays.fill(orders, true); } return orders; }
Example #26
Source File: OperatorDescriptorDual.java From flink with Apache License 2.0 | 5 votes |
protected boolean checkEquivalentFieldPositionsInKeyFields(FieldList fields1, FieldList fields2) { // check number of produced partitioning fields if(fields1.size() != fields2.size()) { return false; } else { return checkEquivalentFieldPositionsInKeyFields(fields1, fields2, fields1.size()); } }
Example #27
Source File: LocalPropertiesFilteringTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testGroupingErased() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->4;2->0"}, null, null, tupleInfo, tupleInfo); LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 2, 3)); LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0); assertNull(filtered.getGroupedFields()); assertNull(filtered.getOrdering()); assertNull(filtered.getUniqueFields()); }
Example #28
Source File: LocalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testSortingPreserved3() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;2"}, null, null, tupleInfo, tupleInfo); Ordering o = new Ordering(); o.appendOrdering(2, IntValue.class, Order.ASCENDING); o.appendOrdering(0, StringValue.class, Order.DESCENDING); o.appendOrdering(5, LongValue.class, Order.DESCENDING); LocalProperties lProps = LocalProperties.forOrdering(o); LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0); FieldList gFields = filtered.getGroupedFields(); Ordering order = filtered.getOrdering(); assertNotNull(gFields); assertEquals(2, gFields.size()); assertTrue(gFields.contains(0)); assertTrue(gFields.contains(2)); assertNotNull(order); assertEquals(2, order.getNumberOfFields()); assertEquals(2, order.getFieldNumber(0).intValue()); assertEquals(0, order.getFieldNumber(1).intValue()); assertEquals(Order.ASCENDING, order.getOrder(0)); assertEquals(Order.DESCENDING, order.getOrder(1)); assertEquals(IntValue.class, order.getType(0)); assertEquals(StringValue.class, order.getType(1)); assertNull(filtered.getUniqueFields()); }
Example #29
Source File: RequestedGlobalProperties.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private boolean checkCompatiblePartitioningFields(GlobalProperties props) { if(this.partitioningFields instanceof FieldList) { // partitioningFields as FieldList requires strict checking! return props.isExactlyPartitionedOnFields((FieldList)this.partitioningFields); } else { return props.isPartitionedOnFields(this.partitioningFields); } }
Example #30
Source File: PartitionPushdownTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPartitioningNotPushedDown() { try { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); @SuppressWarnings("unchecked") DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L)); input .groupBy(0, 1).sum(2) .groupBy(0).sum(1) .output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>()); Plan p = env.createProgramPlan(); OptimizedPlan op = compileNoStats(p); SinkPlanNode sink = op.getDataSinks().iterator().next(); SingleInputPlanNode agg2Reducer = (SingleInputPlanNode) sink.getInput().getSource(); SingleInputPlanNode agg2Combiner = (SingleInputPlanNode) agg2Reducer.getInput().getSource(); SingleInputPlanNode agg1Reducer = (SingleInputPlanNode) agg2Combiner.getInput().getSource(); assertEquals(ShipStrategyType.PARTITION_HASH, agg2Reducer.getInput().getShipStrategy()); assertEquals(new FieldList(0), agg2Reducer.getInput().getShipStrategyKeys()); assertEquals(ShipStrategyType.FORWARD, agg2Combiner.getInput().getShipStrategy()); assertEquals(ShipStrategyType.PARTITION_HASH, agg1Reducer.getInput().getShipStrategy()); assertEquals(new FieldList(0, 1), agg1Reducer.getInput().getShipStrategyKeys()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }