org.apache.flink.api.common.operators.GenericDataSourceBase.SplitDataProperties Java Examples

The following examples show how to use org.apache.flink.api.common.operators.GenericDataSourceBase.SplitDataProperties. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DataSourceNode.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a new DataSourceNode for the given contract.
 * 
 * @param pactContract
 *        The data source contract object.
 */
public DataSourceNode(GenericDataSourceBase<?, ?> pactContract) {
	super(pactContract);
	
	if (pactContract.getUserCodeWrapper().getUserCodeClass() == null) {
		throw new IllegalArgumentException("Input format has not been set.");
	}
	
	if (NonParallelInput.class.isAssignableFrom(pactContract.getUserCodeWrapper().getUserCodeClass())) {
		setParallelism(1);
		this.sequentialInput = true;
	} else {
		this.sequentialInput = false;
	}

	this.replicatedInput = ReplicatingInputFormat.class.isAssignableFrom(
													pactContract.getUserCodeWrapper().getUserCodeClass());

	this.gprops = new GlobalProperties();
	this.lprops = new LocalProperties();

	SplitDataProperties<?> splitProps = pactContract.getSplitDataProperties();

	if(replicatedInput) {
		this.gprops.setFullyReplicated();
		this.lprops = new LocalProperties();
	} else if (splitProps != null) {
		// configure data properties of data source using split properties
		setDataPropertiesFromSplitProperties(splitProps);
	}

}
Example #2
Source File: DataSourceNode.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a new DataSourceNode for the given contract.
 * 
 * @param pactContract
 *        The data source contract object.
 */
public DataSourceNode(GenericDataSourceBase<?, ?> pactContract) {
	super(pactContract);
	
	if (pactContract.getUserCodeWrapper().getUserCodeClass() == null) {
		throw new IllegalArgumentException("Input format has not been set.");
	}
	
	if (NonParallelInput.class.isAssignableFrom(pactContract.getUserCodeWrapper().getUserCodeClass())) {
		setParallelism(1);
		this.sequentialInput = true;
	} else {
		this.sequentialInput = false;
	}

	this.replicatedInput = ReplicatingInputFormat.class.isAssignableFrom(
													pactContract.getUserCodeWrapper().getUserCodeClass());

	this.gprops = new GlobalProperties();
	this.lprops = new LocalProperties();

	SplitDataProperties<?> splitProps = pactContract.getSplitDataProperties();

	if(replicatedInput) {
		this.gprops.setFullyReplicated();
		this.lprops = new LocalProperties();
	} else if (splitProps != null) {
		// configure data properties of data source using split properties
		setDataPropertiesFromSplitProperties(splitProps);
	}

}
Example #3
Source File: DataSourceNode.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a new DataSourceNode for the given contract.
 * 
 * @param pactContract
 *        The data source contract object.
 */
public DataSourceNode(GenericDataSourceBase<?, ?> pactContract) {
	super(pactContract);
	
	if (pactContract.getUserCodeWrapper().getUserCodeClass() == null) {
		throw new IllegalArgumentException("Input format has not been set.");
	}
	
	if (NonParallelInput.class.isAssignableFrom(pactContract.getUserCodeWrapper().getUserCodeClass())) {
		setParallelism(1);
		this.sequentialInput = true;
	} else {
		this.sequentialInput = false;
	}

	this.replicatedInput = ReplicatingInputFormat.class.isAssignableFrom(
													pactContract.getUserCodeWrapper().getUserCodeClass());

	this.gprops = new GlobalProperties();
	this.lprops = new LocalProperties();

	SplitDataProperties<?> splitProps = pactContract.getSplitDataProperties();

	if(replicatedInput) {
		this.gprops.setFullyReplicated();
		this.lprops = new LocalProperties();
	} else if (splitProps != null) {
		// configure data properties of data source using split properties
		setDataPropertiesFromSplitProperties(splitProps);
	}

}
Example #4
Source File: DataSourceNode.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
private void setDataPropertiesFromSplitProperties(SplitDataProperties splitProps) {

		// set global properties
		int[] partitionKeys = splitProps.getSplitPartitionKeys();
		Partitioner<?> partitioner = splitProps.getSplitPartitioner();

		if(partitionKeys != null && partitioner != null) {
			this.gprops.setCustomPartitioned(new FieldList(partitionKeys), partitioner);
		}
		else if(partitionKeys != null) {
			this.gprops.setAnyPartitioning(new FieldList(partitionKeys));
		}
		// set local properties
		int[] groupingKeys = splitProps.getSplitGroupKeys();
		Ordering ordering = splitProps.getSplitOrder();

		// more than one split per source tasks possible.
		// adapt split grouping and sorting
		if(ordering != null) {

			// sorting falls back to grouping because a source can read multiple,
			// randomly assigned splits
			groupingKeys = ordering.getFieldPositions();
		}

		if(groupingKeys != null && partitionKeys != null) {
			// check if grouping is also valid across splits, i.e., whether grouping keys are
			// valid superset of partition keys
			boolean allFieldsIncluded = true;
			for(int i : partitionKeys) {
				boolean fieldIncluded = false;
				for(int j : groupingKeys) {
					if(i == j) {
						fieldIncluded = true;
						break;
					}
				}
				if(!fieldIncluded) {
					allFieldsIncluded = false;
					break;
				}
			}
			if (allFieldsIncluded) {
				this.lprops = LocalProperties.forGrouping(new FieldList(groupingKeys));
			} else {
				this.lprops = new LocalProperties();
			}

		} else {
			this.lprops = new LocalProperties();
		}
	}
Example #5
Source File: DataSourceNode.java    From flink with Apache License 2.0 4 votes vote down vote up 
private void setDataPropertiesFromSplitProperties(SplitDataProperties splitProps) {

		// set global properties
		int[] partitionKeys = splitProps.getSplitPartitionKeys();
		Partitioner<?> partitioner = splitProps.getSplitPartitioner();

		if(partitionKeys != null && partitioner != null) {
			this.gprops.setCustomPartitioned(new FieldList(partitionKeys), partitioner);
		}
		else if(partitionKeys != null) {
			this.gprops.setAnyPartitioning(new FieldList(partitionKeys));
		}
		// set local properties
		int[] groupingKeys = splitProps.getSplitGroupKeys();
		Ordering ordering = splitProps.getSplitOrder();

		// more than one split per source tasks possible.
		// adapt split grouping and sorting
		if(ordering != null) {

			// sorting falls back to grouping because a source can read multiple,
			// randomly assigned splits
			groupingKeys = ordering.getFieldPositions();
		}

		if(groupingKeys != null && partitionKeys != null) {
			// check if grouping is also valid across splits, i.e., whether grouping keys are
			// valid superset of partition keys
			boolean allFieldsIncluded = true;
			for(int i : partitionKeys) {
				boolean fieldIncluded = false;
				for(int j : groupingKeys) {
					if(i == j) {
						fieldIncluded = true;
						break;
					}
				}
				if(!fieldIncluded) {
					allFieldsIncluded = false;
					break;
				}
			}
			if (allFieldsIncluded) {
				this.lprops = LocalProperties.forGrouping(new FieldList(groupingKeys));
			} else {
				this.lprops = new LocalProperties();
			}

		} else {
			this.lprops = new LocalProperties();
		}
	}
Example #6
Source File: DataSourceNode.java    From flink with Apache License 2.0 4 votes vote down vote up 
private void setDataPropertiesFromSplitProperties(SplitDataProperties splitProps) {

		// set global properties
		int[] partitionKeys = splitProps.getSplitPartitionKeys();
		Partitioner<?> partitioner = splitProps.getSplitPartitioner();

		if(partitionKeys != null && partitioner != null) {
			this.gprops.setCustomPartitioned(new FieldList(partitionKeys), partitioner);
		}
		else if(partitionKeys != null) {
			this.gprops.setAnyPartitioning(new FieldList(partitionKeys));
		}
		// set local properties
		int[] groupingKeys = splitProps.getSplitGroupKeys();
		Ordering ordering = splitProps.getSplitOrder();

		// more than one split per source tasks possible.
		// adapt split grouping and sorting
		if(ordering != null) {

			// sorting falls back to grouping because a source can read multiple,
			// randomly assigned splits
			groupingKeys = ordering.getFieldPositions();
		}

		if(groupingKeys != null && partitionKeys != null) {
			// check if grouping is also valid across splits, i.e., whether grouping keys are
			// valid superset of partition keys
			boolean allFieldsIncluded = true;
			for(int i : partitionKeys) {
				boolean fieldIncluded = false;
				for(int j : groupingKeys) {
					if(i == j) {
						fieldIncluded = true;
						break;
					}
				}
				if(!fieldIncluded) {
					allFieldsIncluded = false;
					break;
				}
			}
			if (allFieldsIncluded) {
				this.lprops = LocalProperties.forGrouping(new FieldList(groupingKeys));
			} else {
				this.lprops = new LocalProperties();
			}

		} else {
			this.lprops = new LocalProperties();
		}
	}