org.apache.flink.streaming.api.operators.Output Java Examples

The following examples show how to use org.apache.flink.streaming.api.operators.Output. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StreamSourceOperatorLatencyMetricsTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static <T> void setupSourceOperator(
		StreamSource<T, ?> operator,
		ExecutionConfig executionConfig,
		Environment env,
		ProcessingTimeService timeProvider) {

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(TimeCharacteristic.EventTime);
	cfg.setOperatorID(new OperatorID());

	try {
		MockStreamTask mockTask = new MockStreamTaskBuilder(env)
			.setConfig(cfg)
			.setExecutionConfig(executionConfig)
			.setProcessingTimeService(timeProvider)
			.build();

		operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
Example #2
Source File: DirectedOutput.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
protected Set<Output<StreamRecord<OUT>>> selectOutputs(StreamRecord<OUT> record)  {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = new HashSet<>(selectAllOutputs.length);
	Collections.addAll(selectedOutputs, selectAllOutputs);

	for (OutputSelector<OUT> outputSelector : outputSelectors) {
		Iterable<String> outputNames = outputSelector.select(record.getValue());

		for (String outputName : outputNames) {
			Output<StreamRecord<OUT>>[] outputList = outputMap.get(outputName);
			if (outputList != null) {
				Collections.addAll(selectedOutputs, outputList);
			}
		}
	}

	return selectedOutputs;
}
Example #3
Source File: AsyncWaitOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) {
	super.setup(containingTask, config, output);

	this.inStreamElementSerializer = new StreamElementSerializer<>(
		getOperatorConfig().<IN>getTypeSerializerIn1(getUserCodeClassloader()));

	switch (outputMode) {
		case ORDERED:
			queue = new OrderedStreamElementQueue<>(capacity);
			break;
		case UNORDERED:
			queue = new UnorderedStreamElementQueue<>(capacity);
			break;
		default:
			throw new IllegalStateException("Unknown async mode: " + outputMode + '.');
	}

	this.timestampedCollector = new TimestampedCollector<>(output);
}
Example #4
Source File: DoFnOperator.java    From beam with Apache License 2.0 6 votes vote down vote up 
BufferedOutputManager(
    Output<StreamRecord<WindowedValue<OutputT>>> output,
    TupleTag<OutputT> mainTag,
    Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags,
    Map<TupleTag<?>, Integer> tagsToIds,
    Lock bufferLock,
    PushedBackElementsHandler<KV<Integer, WindowedValue<?>>> pushedBackElementsHandler) {
  this.output = output;
  this.mainTag = mainTag;
  this.tagsToOutputTags = tagsToOutputTags;
  this.tagsToIds = tagsToIds;
  this.bufferLock = bufferLock;
  this.idsToTags = new HashMap<>();
  for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) {
    idsToTags.put(entry.getValue(), entry.getKey());
  }
  this.pushedBackElementsHandler = pushedBackElementsHandler;
}
Example #5
Source File: CopyingDirectedOutput.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public void collect(StreamRecord<OUT> record) {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = selectOutputs(record);

	if (selectedOutputs.isEmpty()) {
		return;
	}

	Iterator<Output<StreamRecord<OUT>>> it = selectedOutputs.iterator();

	while (true) {
		Output<StreamRecord<OUT>> out = it.next();
		if (it.hasNext()) {
			// we don't have the last output
			// perform a shallow copy
			StreamRecord<OUT> shallowCopy = record.copy(record.getValue());
			out.collect(shallowCopy);
		} else {
			// this is the last output
			out.collect(record);
			break;
		}
	}
}
Example #6
Source File: DirectedOutput.java    From flink with Apache License 2.0 6 votes vote down vote up 
protected Set<Output<StreamRecord<OUT>>> selectOutputs(StreamRecord<OUT> record)  {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = new HashSet<>(selectAllOutputs.length);
	Collections.addAll(selectedOutputs, selectAllOutputs);

	for (OutputSelector<OUT> outputSelector : outputSelectors) {
		Iterable<String> outputNames = outputSelector.select(record.getValue());

		for (String outputName : outputNames) {
			Output<StreamRecord<OUT>>[] outputList = outputMap.get(outputName);
			if (outputList != null) {
				Collections.addAll(selectedOutputs, outputList);
			}
		}
	}

	return selectedOutputs;
}
Example #7
Source File: DirectedOutput.java    From flink with Apache License 2.0 6 votes vote down vote up 
protected Set<Output<StreamRecord<OUT>>> selectOutputs(StreamRecord<OUT> record)  {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = new HashSet<>(selectAllOutputs.length);
	Collections.addAll(selectedOutputs, selectAllOutputs);

	for (OutputSelector<OUT> outputSelector : outputSelectors) {
		Iterable<String> outputNames = outputSelector.select(record.getValue());

		for (String outputName : outputNames) {
			Output<StreamRecord<OUT>>[] outputList = outputMap.get(outputName);
			if (outputList != null) {
				Collections.addAll(selectedOutputs, outputList);
			}
		}
	}

	return selectedOutputs;
}
Example #8
Source File: CopyingDirectedOutput.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void collect(StreamRecord<OUT> record) {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = selectOutputs(record);

	if (selectedOutputs.isEmpty()) {
		return;
	}

	Iterator<Output<StreamRecord<OUT>>> it = selectedOutputs.iterator();

	while (true) {
		Output<StreamRecord<OUT>> out = it.next();
		if (it.hasNext()) {
			// we don't have the last output
			// perform a shallow copy
			StreamRecord<OUT> shallowCopy = record.copy(record.getValue());
			out.collect(shallowCopy);
		} else {
			// this is the last output
			out.collect(record);
			break;
		}
	}
}
Example #9
Source File: StreamSourceOperatorLatencyMetricsTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private static <T> void setupSourceOperator(
		StreamSource<T, ?> operator,
		ExecutionConfig executionConfig,
		Environment env,
		ProcessingTimeService timeProvider) {

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(TimeCharacteristic.EventTime);
	cfg.setOperatorID(new OperatorID());

	try {
		MockStreamTask mockTask = new MockStreamTaskBuilder(env)
			.setConfig(cfg)
			.setExecutionConfig(executionConfig)
			.setProcessingTimeService(timeProvider)
			.build();

		operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
Example #10
Source File: StreamTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void run(Object lockingObject,
				StreamStatusMaintainer streamStatusMaintainer,
				Output<StreamRecord<Long>> collector,
				OperatorChain<?, ?> operatorChain) throws Exception {
	while (!canceled) {
		try {
			Thread.sleep(500);
		} catch (InterruptedException ignored) {}
	}
}
Example #11
Source File: DirectedOutput.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void emitWatermark(Watermark mark) {
	watermarkGauge.setCurrentWatermark(mark.getTimestamp());
	for (Output<StreamRecord<OUT>> out : allOutputs) {
		out.emitWatermark(mark);
	}
}
Example #12
Source File: CodeGenOperatorFactory.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public <T extends StreamOperator<OUT>> T createStreamOperator(StreamTask<?, ?> containingTask,
		StreamConfig config, Output<StreamRecord<OUT>> output) {
	return (T) generatedClass.newInstance(containingTask.getUserCodeClassLoader(),
			generatedClass.getReferences(), containingTask, config, output);
}
Example #13
Source File: StreamOutputHandler.java    From bahir-flink with Apache License 2.0 5 votes vote down vote up 
public StreamOutputHandler(TypeInformation<R> typeInfo, AbstractDefinition definition, Output<StreamRecord<R>> output) {
    this.typeInfo = typeInfo;
    this.definition = definition;
    this.output = output;
    this.objectMapper = new ObjectMapper();
    this.objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
}
Example #14
Source File: OperatorChain.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void emitWatermark(Watermark mark) {
	watermarkGauge.setCurrentWatermark(mark.getTimestamp());
	if (streamStatusProvider.getStreamStatus().isActive()) {
		for (Output<StreamRecord<T>> output : outputs) {
			output.emitWatermark(mark);
		}
	}
}
Example #15
Source File: DirectedOutput.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void collect(StreamRecord<OUT> record) {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = selectOutputs(record);

	for (Output<StreamRecord<OUT>> out : selectedOutputs) {
		out.collect(record);
	}
}
Example #16
Source File: SourceOperatorStreamTask.java    From flink with Apache License 2.0 5 votes vote down vote up 
AsyncDataOutputToOutput(
		Output<StreamRecord<T>> output,
		StreamStatusMaintainer streamStatusMaintainer) {
	super(streamStatusMaintainer);

	this.output = checkNotNull(output);
}
Example #17
Source File: StreamSourceOperatorLatencyMetricsTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
private static <T> void setupSourceOperator(
		StreamSource<T, ?> operator,
		ExecutionConfig executionConfig,
		Environment env,
		TimerService timerService) {

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(TimeCharacteristic.EventTime);
	cfg.setOperatorID(new OperatorID());

	try {
		MockStreamTask mockTask = new MockStreamTaskBuilder(env)
			.setConfig(cfg)
			.setExecutionConfig(executionConfig)
			.setTimerService(timerService)
			.build();

		operator.setProcessingTimeService(mockTask.getProcessingTimeServiceFactory().createProcessingTimeService(null));
		operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
Example #18
Source File: DirectedOutput.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public void emitWatermark(Watermark mark) {
	watermarkGauge.setCurrentWatermark(mark.getTimestamp());
	for (Output<StreamRecord<OUT>> out : allOutputs) {
		out.emitWatermark(mark);
	}
}
Example #19
Source File: StreamSourceOperatorWatermarksTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
private static <T> void setupSourceOperator(StreamSource<T, ?> operator,
											TimeCharacteristic timeChar,
											long watermarkInterval,
											final ProcessingTimeService timeProvider) throws Exception {

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setAutoWatermarkInterval(watermarkInterval);

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(timeChar);
	cfg.setOperatorID(new OperatorID());

	Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);

	StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class);
	when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE);

	MockStreamTask mockTask = new MockStreamTaskBuilder(env)
		.setConfig(cfg)
		.setExecutionConfig(executionConfig)
		.setStreamStatusMaintainer(streamStatusMaintainer)
		.setProcessingTimeService(timeProvider)
		.build();

	operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
}
Example #20
Source File: StreamSourceOperatorWatermarksTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
private static <T> MockStreamTask setupSourceOperator(
		StreamSource<T, ?> operator,
		TimeCharacteristic timeChar,
		long watermarkInterval,
		final TimerService timeProvider) throws Exception {

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setAutoWatermarkInterval(watermarkInterval);

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(timeChar);
	cfg.setOperatorID(new OperatorID());

	Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);

	StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class);
	when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE);

	MockStreamTask mockTask = new MockStreamTaskBuilder(env)
		.setConfig(cfg)
		.setExecutionConfig(executionConfig)
		.setStreamStatusMaintainer(streamStatusMaintainer)
		.setTimerService(timeProvider)
		.build();

	operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
	return mockTask;
}
Example #21
Source File: StreamOutputHandler.java    From flink-siddhi with Apache License 2.0 5 votes vote down vote up 
public StreamOutputHandler(String outputStreamId, TypeInformation<R> typeInfo, AbstractDefinition definition, Output<StreamRecord<R>> output) {
    this.outputStreamId = outputStreamId;
    this.typeInfo = typeInfo;
    this.definition = definition;
    this.output = output;
    this.objectMapper = new ObjectMapper();
    this.objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
}
Example #22
Source File: DirectedOutput.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void emitWatermark(Watermark mark) {
	watermarkGauge.setCurrentWatermark(mark.getTimestamp());
	for (Output<StreamRecord<OUT>> out : allOutputs) {
		out.emitWatermark(mark);
	}
}
Example #23
Source File: DirectedOutput.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void collect(StreamRecord<OUT> record) {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = selectOutputs(record);

	for (Output<StreamRecord<OUT>> out : selectedOutputs) {
		out.collect(record);
	}
}
Example #24
Source File: Emitter.java    From flink with Apache License 2.0 5 votes vote down vote up 
public Emitter(
		final Object checkpointLock,
		final Output<StreamRecord<OUT>> output,
		final StreamElementQueue streamElementQueue,
		final OperatorActions operatorActions) {

	this.checkpointLock = Preconditions.checkNotNull(checkpointLock, "checkpointLock");
	this.output = Preconditions.checkNotNull(output, "output");
	this.streamElementQueue = Preconditions.checkNotNull(streamElementQueue, "streamElementQueue");
	this.operatorActions = Preconditions.checkNotNull(operatorActions, "operatorActions");

	this.timestampedCollector = new TimestampedCollector<>(this.output);
	this.running = true;
}
Example #25
Source File: OperatorChain.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void emitWatermark(Watermark mark) {
	watermarkGauge.setCurrentWatermark(mark.getTimestamp());
	if (streamStatusProvider.getStreamStatus().isActive()) {
		for (Output<StreamRecord<T>> output : outputs) {
			output.emitWatermark(mark);
		}
	}
}
Example #26
Source File: IngressRouterOperator.java    From flink-statefun with Apache License 2.0 5 votes vote down vote up 
DownstreamCollector(
    MessageFactoryType messageFactoryType, Output<StreamRecord<Message>> output) {
  this.factory = MessageFactory.forType(messageFactoryType);
  this.output = Objects.requireNonNull(output);
  this.multiLanguagePayloads =
      messageFactoryType == MessageFactoryType.WITH_PROTOBUF_PAYLOADS_MULTILANG;
}
Example #27
Source File: OperatorChain.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void collect(StreamRecord<T> record) {

	for (int i = 0; i < outputs.length - 1; i++) {
		Output<StreamRecord<T>> output = outputs[i];
		StreamRecord<T> shallowCopy = record.copy(record.getValue());
		output.collect(shallowCopy);
	}

	if (outputs.length > 0) {
		// don't copy for the last output
		outputs[outputs.length - 1].collect(record);
	}
}
Example #28
Source File: OperatorChain.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> record) {
	for (int i = 0; i < outputs.length - 1; i++) {
		Output<StreamRecord<T>> output = outputs[i];

		StreamRecord<X> shallowCopy = record.copy(record.getValue());
		output.collect(outputTag, shallowCopy);
	}

	if (outputs.length > 0) {
		// don't copy for the last output
		outputs[outputs.length - 1].collect(outputTag, record);
	}
}
Example #29
Source File: IngressRouterOperator.java    From stateful-functions with Apache License 2.0 5 votes vote down vote up 
DownstreamCollector(
    MessageFactoryType messageFactoryType, Output<StreamRecord<Message>> output) {
  this.factory = MessageFactory.forType(messageFactoryType);
  this.output = Objects.requireNonNull(output);
  this.multiLanguagePayloads =
      messageFactoryType == MessageFactoryType.WITH_PROTOBUF_PAYLOADS_MULTILANG;
}
Example #30
Source File: StreamTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void run(Object lockingObject,
				StreamStatusMaintainer streamStatusMaintainer,
				Output<StreamRecord<Long>> collector,
				OperatorChain<?, ?> operatorChain) throws Exception {
	while (!canceled) {
		try {
			Thread.sleep(500);
		} catch (InterruptedException ignored) {}
	}
}