org.apache.flink.util.SplittableIterator Java Examples

public <T> DataSet<T> getGenerator(Class<? extends Iterable<T>> generatorClass, Class<T> type) {
	SplittableIterator<T> si = new TPCHGeneratorSplittableIterator(scale, env.getParallelism(), generatorClass);
	return env.fromParallelCollection(si, type).name("Generator: "+generatorClass);
}
 

private <X> DataSource<X> fromParallelCollection(SplittableIterator<X> iterator, TypeInformation<X> type, String callLocationName) {
	return new DataSource<>(this, new ParallelIteratorInputFormat<>(iterator), type, callLocationName);
}
 

public ParallelIteratorInputFormat(SplittableIterator<T> iterator) {
	this.source = iterator;
}
 

public FromSplittableIteratorFunction(SplittableIterator<T> iterator) {
	this.fullIterator = iterator;
}
 

private <OUT> DataStreamSource<OUT> fromParallelCollection(SplittableIterator<OUT> iterator, TypeInformation<OUT>
		typeInfo, String operatorName) {
	return addSource(new FromSplittableIteratorFunction<>(iterator), operatorName, typeInfo);
}
 

private <OUT> DataStreamSource<OUT> fromParallelCollection(SplittableIterator<OUT> iterator, TypeInformation<OUT>
		typeInfo, String operatorName) {
	return addSource(new FromSplittableIteratorFunction<>(iterator), operatorName, typeInfo);
}
 

public FromSplittableIteratorFunction(SplittableIterator<T> iterator) {
	this.fullIterator = iterator;
}
 

public FromSplittableIteratorFunction(SplittableIterator<T> iterator) {
	this.fullIterator = iterator;
}
 

private <X> DataSource<X> fromParallelCollection(SplittableIterator<X> iterator, TypeInformation<X> type, String callLocationName) {
	return new DataSource<>(this, new ParallelIteratorInputFormat<>(iterator), type, callLocationName);
}
 

public ParallelIteratorInputFormat(SplittableIterator<T> iterator) {
	this.source = iterator;
}
 

public ParallelIteratorInputFormat(SplittableIterator<T> iterator) {
	this.source = iterator;
}
 

private <X> DataSource<X> fromParallelCollection(SplittableIterator<X> iterator, TypeInformation<X> type, String callLocationName) {
	return new DataSource<>(this, new ParallelIteratorInputFormat<>(iterator), type, callLocationName);
}
 

private <OUT> DataStreamSource<OUT> fromParallelCollection(SplittableIterator<OUT> iterator, TypeInformation<OUT>
		typeInfo, String operatorName) {
	return addSource(new FromSplittableIteratorFunction<>(iterator), operatorName, typeInfo);
}
 

/**
 * Creates a new data set that contains elements in the iterator. The iterator is splittable, allowing the
 * framework to create a parallel data source that returns the elements in the iterator.
 *
 * <p>Because the iterator will remain unmodified until the actual execution happens, the type of data
 * returned by the iterator must be given explicitly in the form of the type information.
 * This method is useful for cases where the type is generic. In that case, the type class
 * (as given in {@link #fromParallelCollection(SplittableIterator, Class)} does not supply all type information.
 *
 * @param iterator The iterator that produces the elements of the data set.
 * @param type The TypeInformation for the produced data set.
 * @return A DataSet representing the elements in the iterator.
 *
 * @see #fromParallelCollection(SplittableIterator, Class)
 */
public <X> DataSource<X> fromParallelCollection(SplittableIterator<X> iterator, TypeInformation<X> type) {
	return fromParallelCollection(iterator, type, Utils.getCallLocationName());
}
 

/**
 * Creates a new data stream that contains elements in the iterator. The iterator is splittable,
 * allowing the framework to create a parallel data stream source that returns the elements in
 * the iterator.
 *
 * <p>Because the iterator will remain unmodified until the actual execution happens, the type
 * of data returned by the iterator must be given explicitly in the form of the type class
 * (this is due to the fact that the Java compiler erases the generic type information).
 *
 * @param iterator
 * 		The iterator that produces the elements of the data stream
 * @param type
 * 		The class of the data produced by the iterator. Must not be a generic class.
 * @param <OUT>
 * 		The type of the returned data stream
 * @return A data stream representing the elements in the iterator
 */
public <OUT> DataStreamSource<OUT> fromParallelCollection(SplittableIterator<OUT> iterator, Class<OUT> type) {
	return fromParallelCollection(iterator, TypeExtractor.getForClass(type));
}
 

/**
 * Creates a new data stream that contains elements in the iterator. The iterator is splittable,
 * allowing the framework to create a parallel data stream source that returns the elements in
 * the iterator.
 *
 * <p>Because the iterator will remain unmodified until the actual execution happens, the type
 * of data returned by the iterator must be given explicitly in the form of the type class
 * (this is due to the fact that the Java compiler erases the generic type information).
 *
 * @param iterator
 * 		The iterator that produces the elements of the data stream
 * @param type
 * 		The class of the data produced by the iterator. Must not be a generic class.
 * @param <OUT>
 * 		The type of the returned data stream
 * @return A data stream representing the elements in the iterator
 */
public <OUT> DataStreamSource<OUT> fromParallelCollection(SplittableIterator<OUT> iterator, Class<OUT> type) {
	return fromParallelCollection(iterator, TypeExtractor.getForClass(type));
}
 

/**
 * Creates a new data stream that contains elements in the iterator. The iterator is splittable,
 * allowing the framework to create a parallel data stream source that returns the elements in
 * the iterator.
 *
 * <p>Because the iterator will remain unmodified until the actual execution happens, the type
 * of data returned by the iterator must be given explicitly in the form of the type
 * information. This method is useful for cases where the type is generic. In that case, the
 * type class (as given in
 * {@link #fromParallelCollection(org.apache.flink.util.SplittableIterator, Class)} does not
 * supply all type information.
 *
 * @param iterator
 * 		The iterator that produces the elements of the data stream
 * @param typeInfo
 * 		The TypeInformation for the produced data stream.
 * @param <OUT>
 * 		The type of the returned data stream
 * @return A data stream representing the elements in the iterator
 */
public <OUT> DataStreamSource<OUT> fromParallelCollection(SplittableIterator<OUT> iterator, TypeInformation<OUT>
		typeInfo) {
	return fromParallelCollection(iterator, typeInfo, "Parallel Collection Source");
}
 

/**
 * Creates a new data set that contains elements in the iterator. The iterator is splittable, allowing the
 * framework to create a parallel data source that returns the elements in the iterator.
 *
 * <p>Because the iterator will remain unmodified until the actual execution happens, the type of data
 * returned by the iterator must be given explicitly in the form of the type class (this is due to the
 * fact that the Java compiler erases the generic type information).
 *
 * @param iterator The iterator that produces the elements of the data set.
 * @param type The class of the data produced by the iterator. Must not be a generic class.
 * @return A DataSet representing the elements in the iterator.
 *
 * @see #fromParallelCollection(SplittableIterator, TypeInformation)
 */
public <X> DataSource<X> fromParallelCollection(SplittableIterator<X> iterator, Class<X> type) {
	return fromParallelCollection(iterator, TypeExtractor.getForClass(type));
}
 

/**
 * Creates a new data stream that contains elements in the iterator. The iterator is splittable,
 * allowing the framework to create a parallel data stream source that returns the elements in
 * the iterator.
 *
 * <p>Because the iterator will remain unmodified until the actual execution happens, the type
 * of data returned by the iterator must be given explicitly in the form of the type
 * information. This method is useful for cases where the type is generic. In that case, the
 * type class (as given in
 * {@link #fromParallelCollection(org.apache.flink.util.SplittableIterator, Class)} does not
 * supply all type information.
 *
 * @param iterator
 * 		The iterator that produces the elements of the data stream
 * @param typeInfo
 * 		The TypeInformation for the produced data stream.
 * @param <OUT>
 * 		The type of the returned data stream
 * @return A data stream representing the elements in the iterator
 */
public <OUT> DataStreamSource<OUT> fromParallelCollection(SplittableIterator<OUT> iterator, TypeInformation<OUT>
		typeInfo) {
	return fromParallelCollection(iterator, typeInfo, "Parallel Collection Source");
}
 

/**
 * Creates a new data set that contains elements in the iterator. The iterator is splittable, allowing the
 * framework to create a parallel data source that returns the elements in the iterator.
 *
 * <p>Because the iterator will remain unmodified until the actual execution happens, the type of data
 * returned by the iterator must be given explicitly in the form of the type class (this is due to the
 * fact that the Java compiler erases the generic type information).
 *
 * @param iterator The iterator that produces the elements of the data set.
 * @param type The class of the data produced by the iterator. Must not be a generic class.
 * @return A DataSet representing the elements in the iterator.
 *
 * @see #fromParallelCollection(SplittableIterator, TypeInformation)
 */
public <X> DataSource<X> fromParallelCollection(SplittableIterator<X> iterator, Class<X> type) {
	return fromParallelCollection(iterator, TypeExtractor.getForClass(type));
}
 

/**
 * Creates a new data set that contains elements in the iterator. The iterator is splittable, allowing the
 * framework to create a parallel data source that returns the elements in the iterator.
 *
 * <p>Because the iterator will remain unmodified until the actual execution happens, the type of data
 * returned by the iterator must be given explicitly in the form of the type information.
 * This method is useful for cases where the type is generic. In that case, the type class
 * (as given in {@link #fromParallelCollection(SplittableIterator, Class)} does not supply all type information.
 *
 * @param iterator The iterator that produces the elements of the data set.
 * @param type The TypeInformation for the produced data set.
 * @return A DataSet representing the elements in the iterator.
 *
 * @see #fromParallelCollection(SplittableIterator, Class)
 */
public <X> DataSource<X> fromParallelCollection(SplittableIterator<X> iterator, TypeInformation<X> type) {
	return fromParallelCollection(iterator, type, Utils.getCallLocationName());
}
 

/**
 * Creates a new data set that contains elements in the iterator. The iterator is splittable, allowing the
 * framework to create a parallel data source that returns the elements in the iterator.
 *
 * <p>Because the iterator will remain unmodified until the actual execution happens, the type of data
 * returned by the iterator must be given explicitly in the form of the type class (this is due to the
 * fact that the Java compiler erases the generic type information).
 *
 * @param iterator The iterator that produces the elements of the data set.
 * @param type The class of the data produced by the iterator. Must not be a generic class.
 * @return A DataSet representing the elements in the iterator.
 *
 * @see #fromParallelCollection(SplittableIterator, TypeInformation)
 */
public <X> DataSource<X> fromParallelCollection(SplittableIterator<X> iterator, Class<X> type) {
	return fromParallelCollection(iterator, TypeExtractor.getForClass(type));
}
 

/**
 * Creates a new data stream that contains elements in the iterator. The iterator is splittable,
 * allowing the framework to create a parallel data stream source that returns the elements in
 * the iterator.
 *
 * <p>Because the iterator will remain unmodified until the actual execution happens, the type
 * of data returned by the iterator must be given explicitly in the form of the type
 * information. This method is useful for cases where the type is generic. In that case, the
 * type class (as given in
 * {@link #fromParallelCollection(org.apache.flink.util.SplittableIterator, Class)} does not
 * supply all type information.
 *
 * @param iterator
 * 		The iterator that produces the elements of the data stream
 * @param typeInfo
 * 		The TypeInformation for the produced data stream.
 * @param <OUT>
 * 		The type of the returned data stream
 * @return A data stream representing the elements in the iterator
 */
public <OUT> DataStreamSource<OUT> fromParallelCollection(SplittableIterator<OUT> iterator, TypeInformation<OUT>
		typeInfo) {
	return fromParallelCollection(iterator, typeInfo, "Parallel Collection Source");
}
 

/**
 * Creates a new data stream that contains elements in the iterator. The iterator is splittable,
 * allowing the framework to create a parallel data stream source that returns the elements in
 * the iterator.
 *
 * <p>Because the iterator will remain unmodified until the actual execution happens, the type
 * of data returned by the iterator must be given explicitly in the form of the type class
 * (this is due to the fact that the Java compiler erases the generic type information).
 *
 * @param iterator
 * 		The iterator that produces the elements of the data stream
 * @param type
 * 		The class of the data produced by the iterator. Must not be a generic class.
 * @param <OUT>
 * 		The type of the returned data stream
 * @return A data stream representing the elements in the iterator
 */
public <OUT> DataStreamSource<OUT> fromParallelCollection(SplittableIterator<OUT> iterator, Class<OUT> type) {
	return fromParallelCollection(iterator, TypeExtractor.getForClass(type));
}
 

/**
 * Creates a new data set that contains elements in the iterator. The iterator is splittable, allowing the
 * framework to create a parallel data source that returns the elements in the iterator.
 *
 * <p>Because the iterator will remain unmodified until the actual execution happens, the type of data
 * returned by the iterator must be given explicitly in the form of the type information.
 * This method is useful for cases where the type is generic. In that case, the type class
 * (as given in {@link #fromParallelCollection(SplittableIterator, Class)} does not supply all type information.
 *
 * @param iterator The iterator that produces the elements of the data set.
 * @param type The TypeInformation for the produced data set.
 * @return A DataSet representing the elements in the iterator.
 *
 * @see #fromParallelCollection(SplittableIterator, Class)
 */
public <X> DataSource<X> fromParallelCollection(SplittableIterator<X> iterator, TypeInformation<X> type) {
	return fromParallelCollection(iterator, type, Utils.getCallLocationName());
}