Java Code Examples for org.apache.flink.streaming.api.TimeCharacteristic#ProcessingTime

public FailingSource(
	@Nonnull EventEmittingGenerator eventEmittingGenerator,
	@Nonnegative int numberOfGeneratorInvocations,
	@Nonnull TimeCharacteristic timeCharacteristic) {
	this.eventEmittingGenerator = eventEmittingGenerator;
	this.running = true;
	this.emitCallCount = 0;
	this.expectedEmitCalls = numberOfGeneratorInvocations;
	this.failureAfterNumElements = numberOfGeneratorInvocations / 2;
	this.checkpointStatus = new AtomicLong(INITIAL);
	this.usingProcessingTime = timeCharacteristic == TimeCharacteristic.ProcessingTime;
}
 

public FailingSource(
	@Nonnull EventEmittingGenerator eventEmittingGenerator,
	@Nonnegative int numberOfGeneratorInvocations,
	@Nonnull TimeCharacteristic timeCharacteristic) {
	this.eventEmittingGenerator = eventEmittingGenerator;
	this.running = true;
	this.emitCallCount = 0;
	this.expectedEmitCalls = numberOfGeneratorInvocations;
	this.failureAfterNumElements = numberOfGeneratorInvocations / 2;
	this.checkpointStatus = new AtomicLong(INITIAL);
	this.usingProcessingTime = timeCharacteristic == TimeCharacteristic.ProcessingTime;
}
 

public ValidatingSink(
	@Nonnull CountUpdater<T> countUpdater,
	@Nonnull ResultChecker resultChecker,
	@Nonnull TimeCharacteristic timeCharacteristic) {

	this.resultChecker = resultChecker;
	this.countUpdater = countUpdater;
	this.usingProcessingTime = TimeCharacteristic.ProcessingTime == timeCharacteristic;
	this.windowCounts = new HashMap<>();
}
 

/**
 * @param siddhiPlan Siddhi CEP  Execution Plan
 */
public AbstractSiddhiOperator(SiddhiOperatorContext siddhiPlan, String operatorName) {
    validate(siddhiPlan);
    this.executionExpression = siddhiPlan.getFinalExecutionPlan();
    this.siddhiPlan = siddhiPlan;
    this.isProcessingTime = this.siddhiPlan.getTimeCharacteristic() == TimeCharacteristic.ProcessingTime;
    this.streamRecordSerializers = new HashMap<>();
    this.operatorName = operatorName;
    registerStreamRecordSerializers();
}
 

/**
 * @param siddhiPlan Siddhi CEP  Execution Plan
 */
public AbstractSiddhiOperator(SiddhiOperatorContext siddhiPlan) {
    validate(siddhiPlan);
    this.siddhiPlan = siddhiPlan;
    this.isProcessingTime = this.siddhiPlan.getTimeCharacteristic() == TimeCharacteristic.ProcessingTime;
    this.streamRecordSerializers = new HashMap<>();

    registerStreamRecordSerializers();
}
 

public FailingSource(
	@Nonnull EventEmittingGenerator eventEmittingGenerator,
	@Nonnegative int numberOfGeneratorInvocations,
	@Nonnull TimeCharacteristic timeCharacteristic) {
	this.eventEmittingGenerator = eventEmittingGenerator;
	this.running = true;
	this.emitCallCount = 0;
	this.expectedEmitCalls = numberOfGeneratorInvocations;
	this.failureAfterNumElements = numberOfGeneratorInvocations / 2;
	this.checkpointStatus = new AtomicLong(INITIAL);
	this.usingProcessingTime = timeCharacteristic == TimeCharacteristic.ProcessingTime;
}
 

public ValidatingSink(
	@Nonnull CountUpdater<T> countUpdater,
	@Nonnull ResultChecker resultChecker,
	@Nonnull TimeCharacteristic timeCharacteristic) {

	this.resultChecker = resultChecker;
	this.countUpdater = countUpdater;
	this.usingProcessingTime = TimeCharacteristic.ProcessingTime == timeCharacteristic;
	this.windowCounts = new HashMap<>();
}
 

/**
 * Creates a data stream containing results of {@link PatternProcessFunction} to fully matching event patterns.
 *
 * @param processFunction function to be applied to matching event sequences
 * @param outTypeInfo output TypeInformation of
 *        {@link PatternProcessFunction#processMatch(Map, PatternProcessFunction.Context, Collector)}
 * @param <OUT> type of output events
 * @return Data stream containing fully matched event sequence with applied {@link PatternProcessFunction}
 */
<OUT, K> SingleOutputStreamOperator<OUT> build(
		final TypeInformation<OUT> outTypeInfo,
		final PatternProcessFunction<IN, OUT> processFunction) {

	checkNotNull(outTypeInfo);
	checkNotNull(processFunction);

	final TypeSerializer<IN> inputSerializer = inputStream.getType().createSerializer(inputStream.getExecutionConfig());
	final boolean isProcessingTime = inputStream.getExecutionEnvironment().getStreamTimeCharacteristic() == TimeCharacteristic.ProcessingTime;

	final boolean timeoutHandling = processFunction instanceof TimedOutPartialMatchHandler;
	final NFACompiler.NFAFactory<IN> nfaFactory = NFACompiler.compileFactory(pattern, timeoutHandling);

	final CepOperator<IN, K, OUT> operator = new CepOperator<>(
		inputSerializer,
		isProcessingTime,
		nfaFactory,
		comparator,
		pattern.getAfterMatchSkipStrategy(),
		processFunction,
		lateDataOutputTag);

	final SingleOutputStreamOperator<OUT> patternStream;
	if (inputStream instanceof KeyedStream) {
		KeyedStream<IN, K> keyedStream = (KeyedStream<IN, K>) inputStream;

		patternStream = keyedStream.transform(
			"CepOperator",
			outTypeInfo,
			operator);
	} else {
		KeySelector<IN, Byte> keySelector = new NullByteKeySelector<>();

		patternStream = inputStream.keyBy(keySelector).transform(
			"GlobalCepOperator",
			outTypeInfo,
			operator
		).forceNonParallel();
	}

	return patternStream;
}
 

/**
 * Creates a data stream containing results of {@link PatternProcessFunction} to fully matching event patterns.
 *
 * @param processFunction function to be applied to matching event sequences
 * @param outTypeInfo output TypeInformation of
 *        {@link PatternProcessFunction#processMatch(Map, PatternProcessFunction.Context, Collector)}
 * @param <OUT> type of output events
 * @return Data stream containing fully matched event sequence with applied {@link PatternProcessFunction}
 */
<OUT, K> SingleOutputStreamOperator<OUT> build(
		final TypeInformation<OUT> outTypeInfo,
		final PatternProcessFunction<IN, OUT> processFunction) {

	checkNotNull(outTypeInfo);
	checkNotNull(processFunction);

	final TypeSerializer<IN> inputSerializer = inputStream.getType().createSerializer(inputStream.getExecutionConfig());
	final boolean isProcessingTime = inputStream.getExecutionEnvironment().getStreamTimeCharacteristic() == TimeCharacteristic.ProcessingTime;

	final boolean timeoutHandling = processFunction instanceof TimedOutPartialMatchHandler;
	final NFACompiler.NFAFactory<IN> nfaFactory = NFACompiler.compileFactory(pattern, timeoutHandling);

	final CepOperator<IN, K, OUT> operator = new CepOperator<>(
		inputSerializer,
		isProcessingTime,
		nfaFactory,
		comparator,
		pattern.getAfterMatchSkipStrategy(),
		processFunction,
		lateDataOutputTag);

	final SingleOutputStreamOperator<OUT> patternStream;
	if (inputStream instanceof KeyedStream) {
		KeyedStream<IN, K> keyedStream = (KeyedStream<IN, K>) inputStream;

		patternStream = keyedStream.transform(
			"CepOperator",
			outTypeInfo,
			operator);
	} else {
		KeySelector<IN, Byte> keySelector = new NullByteKeySelector<>();

		patternStream = inputStream.keyBy(keySelector).transform(
			"GlobalCepOperator",
			outTypeInfo,
			operator
		).forceNonParallel();
	}

	return patternStream;
}
 

/**
 * Creates a data stream containing results of {@link PatternProcessFunction} to fully matching event patterns.
 *
 * @param processFunction function to be applied to matching event sequences
 * @param outTypeInfo output TypeInformation of
 *        {@link PatternProcessFunction#processMatch(Map, PatternProcessFunction.Context, Collector)}
 * @param <OUT> type of output events
 * @return Data stream containing fully matched event sequence with applied {@link PatternProcessFunction}
 */
<OUT, K> SingleOutputStreamOperator<OUT> build(
		final TypeInformation<OUT> outTypeInfo,
		final PatternProcessFunction<IN, OUT> processFunction) {

	checkNotNull(outTypeInfo);
	checkNotNull(processFunction);

	final TypeSerializer<IN> inputSerializer = inputStream.getType().createSerializer(inputStream.getExecutionConfig());
	final boolean isProcessingTime = inputStream.getExecutionEnvironment().getStreamTimeCharacteristic() == TimeCharacteristic.ProcessingTime;

	final boolean timeoutHandling = processFunction instanceof TimedOutPartialMatchHandler;
	final NFACompiler.NFAFactory<IN> nfaFactory = NFACompiler.compileFactory(pattern, timeoutHandling);

	final CepOperator<IN, K, OUT> operator = new CepOperator<>(
		inputSerializer,
		isProcessingTime,
		nfaFactory,
		comparator,
		pattern.getAfterMatchSkipStrategy(),
		processFunction,
		lateDataOutputTag);

	final SingleOutputStreamOperator<OUT> patternStream;
	if (inputStream instanceof KeyedStream) {
		KeyedStream<IN, K> keyedStream = (KeyedStream<IN, K>) inputStream;

		patternStream = keyedStream.transform(
			"CepOperator",
			outTypeInfo,
			operator);
	} else {
		KeySelector<IN, Byte> keySelector = new NullByteKeySelector<>();

		patternStream = inputStream.keyBy(keySelector).transform(
			"GlobalCepOperator",
			outTypeInfo,
			operator
		).forceNonParallel();
	}

	return patternStream;
}
 

/**
 * Windows this {@code KeyedStream} into sliding time windows.
 *
 * <p>This is a shortcut for either {@code .window(SlidingEventTimeWindows.of(size, slide))} or
 * {@code .window(SlidingProcessingTimeWindows.of(size, slide))} depending on the time
 * characteristic set using
 * {@link org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#setStreamTimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic)}
 *
 * @param size The size of the window.
 */
public WindowedStream<T, KEY, TimeWindow> timeWindow(Time size, Time slide) {
	if (environment.getStreamTimeCharacteristic() == TimeCharacteristic.ProcessingTime) {
		return window(SlidingProcessingTimeWindows.of(size, slide));
	} else {
		return window(SlidingEventTimeWindows.of(size, slide));
	}
}
 

/**
 * Windows this {@code DataStream} into sliding time windows.
 *
 * <p>This is a shortcut for either {@code .window(SlidingEventTimeWindows.of(size, slide))} or
 * {@code .window(SlidingProcessingTimeWindows.of(size, slide))} depending on the time characteristic
 * set using
 * {@link org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#setStreamTimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic)}
 *
 * <p>Note: This operation is inherently non-parallel since all elements have to pass through
 * the same operator instance.
 *
 * @param size The size of the window.
 */
public AllWindowedStream<T, TimeWindow> timeWindowAll(Time size, Time slide) {
	if (environment.getStreamTimeCharacteristic() == TimeCharacteristic.ProcessingTime) {
		return windowAll(SlidingProcessingTimeWindows.of(size, slide));
	} else {
		return windowAll(SlidingEventTimeWindows.of(size, slide));
	}
}
 

/**
 * Sets the time characteristic for all streams create from this environment, e.g., processing
 * time, event time, or ingestion time.
 *
 * <p>If you set the characteristic to IngestionTime of EventTime this will set a default
 * watermark update interval of 200 ms. If this is not applicable for your application
 * you should change it using {@link ExecutionConfig#setAutoWatermarkInterval(long)}.
 *
 * @param characteristic The time characteristic.
 */
@PublicEvolving
public void setStreamTimeCharacteristic(TimeCharacteristic characteristic) {
	this.timeCharacteristic = Preconditions.checkNotNull(characteristic);
	if (characteristic == TimeCharacteristic.ProcessingTime) {
		getConfig().setAutoWatermarkInterval(0);
	} else {
		getConfig().setAutoWatermarkInterval(200);
	}
}
 

/**
 * Windows this {@code DataStream} into tumbling time windows.
 *
 * <p>This is a shortcut for either {@code .window(TumblingEventTimeWindows.of(size))} or
 * {@code .window(TumblingProcessingTimeWindows.of(size))} depending on the time characteristic
 * set using
 *
 * <p>Note: This operation is inherently non-parallel since all elements have to pass through
 * the same operator instance.
 *
 * {@link org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#setStreamTimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic)}
 *
 * @param size The size of the window.
 */
public AllWindowedStream<T, TimeWindow> timeWindowAll(Time size) {
	if (environment.getStreamTimeCharacteristic() == TimeCharacteristic.ProcessingTime) {
		return windowAll(TumblingProcessingTimeWindows.of(size));
	} else {
		return windowAll(TumblingEventTimeWindows.of(size));
	}
}
 

/**
 * Windows this {@code DataStream} into sliding time windows.
 *
 * <p>This is a shortcut for either {@code .window(SlidingEventTimeWindows.of(size, slide))} or
 * {@code .window(SlidingProcessingTimeWindows.of(size, slide))} depending on the time characteristic
 * set using
 * {@link org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#setStreamTimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic)}
 *
 * <p>Note: This operation is inherently non-parallel since all elements have to pass through
 * the same operator instance.
 *
 * @param size The size of the window.
 */
public AllWindowedStream<T, TimeWindow> timeWindowAll(Time size, Time slide) {
	if (environment.getStreamTimeCharacteristic() == TimeCharacteristic.ProcessingTime) {
		return windowAll(SlidingProcessingTimeWindows.of(size, slide));
	} else {
		return windowAll(SlidingEventTimeWindows.of(size, slide));
	}
}
 

/**
 * Windows this {@code KeyedStream} into tumbling time windows.
 *
 * <p>This is a shortcut for either {@code .window(TumblingEventTimeWindows.of(size))} or
 * {@code .window(TumblingProcessingTimeWindows.of(size))} depending on the time characteristic
 * set using
 * {@link org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#setStreamTimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic)}
 *
 * @param size The size of the window.
 */
public WindowedStream<T, KEY, TimeWindow> timeWindow(Time size) {
	if (environment.getStreamTimeCharacteristic() == TimeCharacteristic.ProcessingTime) {
		return window(TumblingProcessingTimeWindows.of(size));
	} else {
		return window(TumblingEventTimeWindows.of(size));
	}
}
 

/**
 * Windows this {@code KeyedStream} into tumbling time windows.
 *
 * <p>This is a shortcut for either {@code .window(TumblingEventTimeWindows.of(size))} or
 * {@code .window(TumblingProcessingTimeWindows.of(size))} depending on the time characteristic
 * set using
 * {@link org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#setStreamTimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic)}
 *
 * @param size The size of the window.
 */
public WindowedStream<T, KEY, TimeWindow> timeWindow(Time size) {
	if (environment.getStreamTimeCharacteristic() == TimeCharacteristic.ProcessingTime) {
		return window(TumblingProcessingTimeWindows.of(size));
	} else {
		return window(TumblingEventTimeWindows.of(size));
	}
}
 

/**
 * Sets the time characteristic for all streams create from this environment, e.g., processing
 * time, event time, or ingestion time.
 *
 * <p>If you set the characteristic to IngestionTime of EventTime this will set a default
 * watermark update interval of 200 ms. If this is not applicable for your application
 * you should change it using {@link ExecutionConfig#setAutoWatermarkInterval(long)}.
 *
 * @param characteristic The time characteristic.
 */
@PublicEvolving
public void setStreamTimeCharacteristic(TimeCharacteristic characteristic) {
	this.timeCharacteristic = Preconditions.checkNotNull(characteristic);
	if (characteristic == TimeCharacteristic.ProcessingTime) {
		getConfig().setAutoWatermarkInterval(0);
	} else {
		getConfig().setAutoWatermarkInterval(200);
	}
}
 

/**
 * Windows this {@code DataStream} into sliding time windows.
 *
 * <p>This is a shortcut for either {@code .window(SlidingEventTimeWindows.of(size, slide))} or
 * {@code .window(SlidingProcessingTimeWindows.of(size, slide))} depending on the time characteristic
 * set using
 * {@link org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#setStreamTimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic)}
 *
 * <p>Note: This operation is inherently non-parallel since all elements have to pass through
 * the same operator instance.
 *
 * @param size The size of the window.
 */
public AllWindowedStream<T, TimeWindow> timeWindowAll(Time size, Time slide) {
	if (environment.getStreamTimeCharacteristic() == TimeCharacteristic.ProcessingTime) {
		return windowAll(SlidingProcessingTimeWindows.of(size, slide));
	} else {
		return windowAll(SlidingEventTimeWindows.of(size, slide));
	}
}
 

/**
 * Windows this {@code DataStream} into tumbling time windows.
 *
 * <p>This is a shortcut for either {@code .window(TumblingEventTimeWindows.of(size))} or
 * {@code .window(TumblingProcessingTimeWindows.of(size))} depending on the time characteristic
 * set using
 *
 * <p>Note: This operation is inherently non-parallel since all elements have to pass through
 * the same operator instance.
 *
 * {@link org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#setStreamTimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic)}
 *
 * @param size The size of the window.
 */
public AllWindowedStream<T, TimeWindow> timeWindowAll(Time size) {
	if (environment.getStreamTimeCharacteristic() == TimeCharacteristic.ProcessingTime) {
		return windowAll(TumblingProcessingTimeWindows.of(size));
	} else {
		return windowAll(TumblingEventTimeWindows.of(size));
	}
}