cascading.pipe.GroupBy Java Examples

public BoundaryGroupByMatcher() {
	super(
			(new ExpressionGraph()).
					arc(
							new TypeExpression(ElementCapture.Primary, Boundary.class, TypeExpression.Topo.LinearOut),
							ScopeExpression.ALL,
							new TypeExpression(ElementCapture.Secondary, GroupBy.class, TypeExpression.Topo.LinearIn)
			)
	);
}
 

@Override
public void open(Configuration config) {

	this.calledPrepare = false;

	try {

		currentProcess = new FlinkFlowProcess(FlinkConfigConverter.toHadoopConfig(config), getRuntimeContext(), flowNode.getID());

		Set<FlowElement> sources = flowNode.getSourceElements();
		if(sources.size() != 1) {
			throw new RuntimeException("FlowNode for GroupByReducer may only have a single source");
		}
		FlowElement sourceElement = sources.iterator().next();
		if(!(sourceElement instanceof GroupBy)) {
			throw new RuntimeException("Source of GroupByReducer must be a GroupBy");
		}
		GroupBy source = (GroupBy)sourceElement;

		streamGraph = new GroupByStreamGraph( currentProcess, flowNode, source );
		groupSource = this.streamGraph.getGroupSource();

		for( Duct head : streamGraph.getHeads() ) {
			LOG.info("sourcing from: " + ((ElementDuct) head).getFlowElement());
		}

		for( Duct tail : streamGraph.getTails() ) {
			LOG.info("sinking to: " + ((ElementDuct) tail).getFlowElement());
		}
	}
	catch( Throwable throwable ) {

		if( throwable instanceof CascadingException) {
			throw (CascadingException) throwable;
		}

		throw new FlowException( "internal error during GroupByReducer configuration", throwable );
	}

}
 

private DataSet<Tuple> translateGroupBy(DataSet<Tuple> input, FlowNode node, int dop) {

		GroupBy groupBy = (GroupBy) node.getSourceElements().iterator().next();

		Scope outScope = getOutScope(node);
		List<Scope> inScopes = getInputScopes(node, groupBy);

		Fields outFields;
		if(outScope.isEvery()) {
			outFields = outScope.getOutGroupingFields();
		}
		else {
			outFields = outScope.getOutValuesFields();
		}
		registerKryoTypes(outFields);

		// get input scope
		Scope inScope = inScopes.get(0);

		// get grouping keys
		Fields groupKeyFields = groupBy.getKeySelectors().get(inScope.getName());
		// get group sorting keys
		Fields sortKeyFields = groupBy.getSortingSelectors().get(inScope.getName());

		String[] groupKeys = registerKeyFields(input, groupKeyFields);
		String[] sortKeys = null;
		if (sortKeyFields != null) {
			sortKeys = registerKeyFields(input, sortKeyFields);
		}
		Order sortOrder = groupBy.isSortReversed() ? Order.DESCENDING : Order.ASCENDING;

		if(sortOrder == Order.DESCENDING) {
			// translate groupBy with inverse sort order
			return translateInverseSortedGroupBy(input, node, dop, groupKeys, sortKeys, outFields);
		}
		else if(groupKeys == null || groupKeys.length == 0) {
			// translate key-less (global) groupBy
			return translateGlobalGroupBy(input, node, dop, sortKeys, sortOrder, outFields);
		}
		else {

			UnsortedGrouping<Tuple> grouping = input
					.groupBy(groupKeys);

			if(sortKeys != null && sortKeys.length > 0) {
				// translate groupBy with group sorting

				SortedGrouping<Tuple> sortedGrouping = grouping
						.sortGroup(sortKeys[0], Order.ASCENDING);
				for(int i=1; i<sortKeys.length; i++) {
					sortedGrouping = sortedGrouping
							.sortGroup(sortKeys[i], Order.DESCENDING);
				}

				return sortedGrouping
						.reduceGroup(new GroupByReducer(node))
						.returns(new TupleTypeInfo(outFields))
						.withParameters(this.getFlinkNodeConfig(node))
						.setParallelism(dop)
						.name("reduce-" + node.getID());
			}
			else {
				// translate groupBy without group sorting

				return grouping
						.reduceGroup(new GroupByReducer(node))
						.returns(new TupleTypeInfo(outFields))
						.withParameters(this.getFlinkNodeConfig(node))
						.setParallelism(dop)
						.name("reduce-" + node.getID());
			}
		}

	}
 

@Override
protected Gate createGroupByGate(GroupBy groupBy, IORole ioRole) {
	throw new UnsupportedOperationException("Cannot create a GroupBy gate in a HashJoinMapperStreamGraph");
}
 

@Override
protected Gate createGroupByGate(GroupBy groupBy, IORole ioRole) {
	throw new UnsupportedOperationException("Cannot create a GroupBy gate in a HashJoinStreamGraph");
}
 

private void buildGraph( GroupBy groupBy, FlowProcess flowProcess ) {

		this.sourceStage = new GroupByInGate(flowProcess, groupBy, IORole.source);
		addHead( sourceStage );
		handleDuct( groupBy, sourceStage );
	}
 

@Override
protected Gate createGroupByGate(GroupBy groupBy, IORole ioRole) {
	throw new UnsupportedOperationException("Cannot create a GroupBy gate in a GroupByStreamGraph");
}
 

public GroupByInGate(FlowProcess flowProcess, GroupBy splice, IORole ioRole) {
	super(flowProcess, splice, ioRole);

	this.isBufferJoin = splice.getJoiner() instanceof BufferJoin;
}
 

@Override
protected Gate createGroupByGate(GroupBy groupBy, IORole ioRole) {
	throw new UnsupportedOperationException("Cannot create a GroupBy gate in a SinkStreamGraph");
}
 

@Override
protected Gate createGroupByGate(GroupBy groupBy, IORole ioRole) {
	throw new UnsupportedOperationException("Cannot create a GroupBy gate in a MapStreamGraph.");
}
 

@Override
protected Gate createGroupByGate(GroupBy element, IORole role) {
	throw new UnsupportedOperationException("SourceStreamGraph may not have a GroupByGate");
}
 

@Override
protected Gate createGroupByGate(GroupBy groupBy, IORole ioRole) {
	throw new UnsupportedOperationException("Cannot create a GroupBy gate in a CoGroupBufferReduceStreamGraph");
}
 

@Override
protected Gate createGroupByGate(GroupBy groupBy, IORole ioRole) {
	throw new UnsupportedOperationException("Cannot create a GroupBy gate in a CoGroupReduceStreamGraph");
}
 

public GroupByStreamGraph(FlinkFlowProcess flowProcess, FlowNode node, GroupBy groupBy) {

		super(flowProcess, node);

		buildGraph(groupBy, flowProcess);

		setTraps();
		setScopes();

		printGraph( node.getID(), "groupby", flowProcess.getCurrentSliceNum() );
		bind();
	}
 

public GroupByGraph() {

			super(SearchOrder.ReverseTopological, new FlowElementExpression(ElementCapture.Primary, GroupBy.class));

		}