org.apache.flink.streaming.api.collector.selector.OutputSelector Java Examples

protected StreamNode addNode(Integer vertexID,
	@Nullable String slotSharingGroup,
	@Nullable String coLocationGroup,
	Class<? extends AbstractInvokable> vertexClass,
	StreamOperatorFactory<?> operatorFactory,
	String operatorName) {

	if (streamNodes.containsKey(vertexID)) {
		throw new RuntimeException("Duplicate vertexID " + vertexID);
	}

	StreamNode vertex = new StreamNode(
		vertexID,
		slotSharingGroup,
		coLocationGroup,
		operatorFactory,
		operatorName,
		new ArrayList<OutputSelector<?>>(),
		vertexClass);

	streamNodes.put(vertexID, vertex);

	return vertex;
}
 

/**
 * Users of the test harness can call this utility method to setup the stream config
 * if there will only be a single operator to be tested. The method will setup the
 * outgoing network connection for the operator.
 *
 * <p>For more advanced test cases such as testing chains of multiple operators with the harness,
 * please manually configure the stream config.
 */
public void setupOutputForSingletonOperatorChain() {
	Preconditions.checkState(!setupCalled, "This harness was already setup.");
	setupCalled = true;
	streamConfig.setChainStart();
	streamConfig.setBufferTimeout(0);
	streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime);
	streamConfig.setOutputSelectors(Collections.<OutputSelector<?>>emptyList());
	streamConfig.setNumberOfOutputs(1);
	streamConfig.setTypeSerializerOut(outputSerializer);
	streamConfig.setVertexID(0);
	streamConfig.setOperatorID(new OperatorID(4711L, 123L));

	StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() {
		private static final long serialVersionUID = 1L;
	};

	List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>();
	StreamNode sourceVertexDummy = new StreamNode(null, 0, "group", null, dummyOperator, "source dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);
	StreamNode targetVertexDummy = new StreamNode(null, 1, "group", null, dummyOperator, "target dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);

	outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new LinkedList<String>(), new BroadcastPartitioner<Object>(), null /* output tag */));

	streamConfig.setOutEdgesInOrder(outEdgesInOrder);
	streamConfig.setNonChainedOutputs(outEdgesInOrder);
}
 

protected StreamNode addNode(
		Integer vertexID,
		@Nullable String slotSharingGroup,
		@Nullable String coLocationGroup,
		Class<? extends AbstractInvokable> vertexClass,
		StreamOperatorFactory<?> operatorFactory,
		String operatorName) {

	if (streamNodes.containsKey(vertexID)) {
		throw new RuntimeException("Duplicate vertexID " + vertexID);
	}

	StreamNode vertex = new StreamNode(
			vertexID,
			slotSharingGroup,
			coLocationGroup,
			operatorFactory,
			operatorName,
			new ArrayList<OutputSelector<?>>(),
			vertexClass);

	streamNodes.put(vertexID, vertex);

	return vertex;
}
 

/**
 * Users of the test harness can call this utility method to setup the stream config
 * if there will only be a single operator to be tested. The method will setup the
 * outgoing network connection for the operator.
 *
 * <p>For more advanced test cases such as testing chains of multiple operators with the harness,
 * please manually configure the stream config.
 */
public void setupOutputForSingletonOperatorChain() {
	Preconditions.checkState(!setupCalled, "This harness was already setup.");
	setupCalled = true;
	streamConfig.setChainStart();
	streamConfig.setBufferTimeout(0);
	streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime);
	streamConfig.setOutputSelectors(Collections.<OutputSelector<?>>emptyList());
	streamConfig.setNumberOfOutputs(1);
	streamConfig.setTypeSerializerOut(outputSerializer);
	streamConfig.setVertexID(0);
	streamConfig.setOperatorID(new OperatorID(4711L, 123L));

	StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() {
		private static final long serialVersionUID = 1L;
	};

	List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>();
	StreamNode sourceVertexDummy = new StreamNode(0, "group", null, dummyOperator, "source dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);
	StreamNode targetVertexDummy = new StreamNode(1, "group", null, dummyOperator, "target dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);

	outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new LinkedList<String>(), new BroadcastPartitioner<Object>(), null /* output tag */));

	streamConfig.setOutEdgesInOrder(outEdgesInOrder);
	streamConfig.setNonChainedOutputs(outEdgesInOrder);
}
 

private static void splitFunction(StreamExecutionEnvironment environment) {
    DataStreamSource<Long> dataStreamSource = environment.addSource(new JavaCustomNonParallelSourceFunction());

    SplitStream<Long> splitStream = dataStreamSource.split(new OutputSelector<Long>() {
        @Override
        public Iterable<String> select(Long value) {
            List<String> list = new ArrayList<>();
            if (value % 2 == 0) {
                list.add("event");
            } else {
                list.add("odd");
            }
            return list;
        }
    });

    splitStream.select("odd").print().setParallelism(1);
}
 

public StreamNode(StreamExecutionEnvironment env,
	Integer id,
	String slotSharingGroup,
	@Nullable String coLocationGroup,
	StreamOperator<?> operator,
	String operatorName,
	List<OutputSelector<?>> outputSelector,
	Class<? extends AbstractInvokable> jobVertexClass) {

	this.env = env;
	this.id = id;
	this.operatorName = operatorName;
	this.operator = operator;
	this.outputSelectors = outputSelector;
	this.jobVertexClass = jobVertexClass;
	this.slotSharingGroup = slotSharingGroup;
	this.coLocationGroup = coLocationGroup;
}
 

public StreamNode(
		Integer id,
		@Nullable String slotSharingGroup,
		@Nullable String coLocationGroup,
		StreamOperatorFactory<?> operatorFactory,
		String operatorName,
		List<OutputSelector<?>> outputSelector,
		Class<? extends AbstractInvokable> jobVertexClass) {
	this.id = id;
	this.operatorName = operatorName;
	this.operatorFactory = operatorFactory;
	this.outputSelectors = outputSelector;
	this.jobVertexClass = jobVertexClass;
	this.slotSharingGroup = slotSharingGroup;
	this.coLocationGroup = coLocationGroup;
}
 

public StreamNode(
	Integer id,
	@Nullable String slotSharingGroup,
	@Nullable String coLocationGroup,
	StreamOperatorFactory<?> operatorFactory,
	String operatorName,
	List<OutputSelector<?>> outputSelector,
	Class<? extends AbstractInvokable> jobVertexClass) {

	this.id = id;
	this.operatorName = operatorName;
	this.operatorFactory = operatorFactory;
	this.outputSelectors = outputSelector;
	this.jobVertexClass = jobVertexClass;
	this.slotSharingGroup = slotSharingGroup;
	this.coLocationGroup = coLocationGroup;
}
 

protected StreamNode addNode(Integer vertexID,
	String slotSharingGroup,
	@Nullable String coLocationGroup,
	Class<? extends AbstractInvokable> vertexClass,
	StreamOperator<?> operatorObject,
	String operatorName) {

	if (streamNodes.containsKey(vertexID)) {
		throw new RuntimeException("Duplicate vertexID " + vertexID);
	}

	StreamNode vertex = new StreamNode(environment,
		vertexID,
		slotSharingGroup,
		coLocationGroup,
		operatorObject,
		operatorName,
		new ArrayList<OutputSelector<?>>(),
		vertexClass);

	streamNodes.put(vertexID, vertex);

	return vertex;
}
 

public <T> List<OutputSelector<T>> getOutputSelectors(ClassLoader userCodeClassloader) {
	try {
		List<OutputSelector<T>> selectors =
				InstantiationUtil.readObjectFromConfig(this.config, OUTPUT_SELECTOR_WRAPPER, userCodeClassloader);
		return selectors == null ? Collections.<OutputSelector<T>>emptyList() : selectors;

	} catch (Exception e) {
		throw new StreamTaskException("Could not read output selectors", e);
	}
}
 

public void setOutputSelectors(List<OutputSelector<?>> outputSelectors) {
	try {
		InstantiationUtil.writeObjectToConfig(outputSelectors, this.config, OUTPUT_SELECTOR_WRAPPER);
	} catch (IOException e) {
		throw new StreamTaskException("Could not serialize output selectors", e);
	}
}
 

@VisibleForTesting
public StreamNode(
		Integer id,
		@Nullable String slotSharingGroup,
		@Nullable String coLocationGroup,
		StreamOperator<?> operator,
		String operatorName,
		List<OutputSelector<?>> outputSelector,
		Class<? extends AbstractInvokable> jobVertexClass) {
	this(id, slotSharingGroup, coLocationGroup, SimpleOperatorFactory.of(operator),
			operatorName, outputSelector, jobVertexClass);
}
 

/**
 * Creates a new {@code SplitTransformation} from the given input and {@code OutputSelector}.
 *
 * @param input The input {@code Transformation}
 * @param outputSelector The output selector
 */
public SplitTransformation(
	Transformation<T> input,
		OutputSelector<T> outputSelector) {
	super("Split", input.getOutputType(), input.getParallelism());
	this.input = input;
	this.outputSelector = outputSelector;
}
 

@Test
public void testGetOutputs() {
	OutputSelector<Tuple1<Integer>> selector = new MyOutputSelector();
	List<String> expectedOutputs = new ArrayList<String>();
	expectedOutputs.add("0");
	expectedOutputs.add("1");
	assertEquals(expectedOutputs, selector.select(new Tuple1<Integer>(2)));
	expectedOutputs.add("2");
	assertEquals(expectedOutputs, selector.select(new Tuple1<Integer>(3)));
}
 

/**
 * Creates a new {@code SplitTransformation} from the given input and {@code OutputSelector}.
 *
 * @param input The input {@code StreamTransformation}
 * @param outputSelector The output selector
 */
public SplitTransformation(StreamTransformation<T> input,
		OutputSelector<T> outputSelector) {
	super("Split", input.getOutputType(), input.getParallelism());
	this.input = input;
	this.outputSelector = outputSelector;
}
 

public void setOutputSelectors(List<OutputSelector<?>> outputSelectors) {
	try {
		InstantiationUtil.writeObjectToConfig(outputSelectors, this.config, OUTPUT_SELECTOR_WRAPPER);
	} catch (IOException e) {
		throw new StreamTaskException("Could not serialize output selectors", e);
	}
}
 

@Override
public SplitStream<T> split(OutputSelector<T> outputSelector) {
	if (requestedSideOutputs.isEmpty()) {
		wasSplitApplied = true;
		return super.split(outputSelector);
	} else {
		throw new UnsupportedOperationException("getSideOutput() and split() may not be called on the same DataStream. " +
			"As a work-around, please add a no-op map function before the split() call.");
	}
}
 

public <T> void addOutputSelector(Integer vertexID, OutputSelector<T> outputSelector) {
	if (virtualPartitionNodes.containsKey(vertexID)) {
		addOutputSelector(virtualPartitionNodes.get(vertexID).f0, outputSelector);
	} else if (virtualSelectNodes.containsKey(vertexID)) {
		addOutputSelector(virtualSelectNodes.get(vertexID).f0, outputSelector);
	} else {
		getStreamNode(vertexID).addOutputSelector(outputSelector);

		if (LOG.isDebugEnabled()) {
			LOG.debug("Outputselector set for {}", vertexID);
		}
	}

}
 

public <T> List<OutputSelector<T>> getOutputSelectors(ClassLoader userCodeClassloader) {
	try {
		List<OutputSelector<T>> selectors =
				InstantiationUtil.readObjectFromConfig(this.config, OUTPUT_SELECTOR_WRAPPER, userCodeClassloader);
		return selectors == null ? Collections.<OutputSelector<T>>emptyList() : selectors;

	} catch (Exception e) {
		throw new StreamTaskException("Could not read output selectors", e);
	}
}
 

public <T> void addOutputSelector(Integer vertexID, OutputSelector<T> outputSelector) {
	if (virtualPartitionNodes.containsKey(vertexID)) {
		addOutputSelector(virtualPartitionNodes.get(vertexID).f0, outputSelector);
	} else if (virtualSelectNodes.containsKey(vertexID)) {
		addOutputSelector(virtualSelectNodes.get(vertexID).f0, outputSelector);
	} else {
		getStreamNode(vertexID).addOutputSelector(outputSelector);

		if (LOG.isDebugEnabled()) {
			LOG.debug("Outputselector set for {}", vertexID);
		}
	}

}
 

@Test
public void testGetOutputs() {
	OutputSelector<Tuple1<Integer>> selector = new MyOutputSelector();
	List<String> expectedOutputs = new ArrayList<String>();
	expectedOutputs.add("0");
	expectedOutputs.add("1");
	assertEquals(expectedOutputs, selector.select(new Tuple1<Integer>(2)));
	expectedOutputs.add("2");
	assertEquals(expectedOutputs, selector.select(new Tuple1<Integer>(3)));
}
 

public static void main(String[] args) throws Exception {
    final ParameterTool params = ParameterTool.fromArgs(args);
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setGlobalJobParameters(params);

    DataStreamSource<MetricEvent> data = KafkaConfigUtil.buildSource(env);  //从 Kafka 获取到所有的数据流
    SplitStream<MetricEvent> splitData = data.split(new OutputSelector<MetricEvent>() {
        @Override
        public Iterable<String> select(MetricEvent metricEvent) {
            List<String> tags = new ArrayList<>();
            String type = metricEvent.getTags().get("type");
            switch (type) {
                case "machine":
                    tags.add("machine");
                    break;
                case "docker":
                    tags.add("docker");
                    break;
                case "application":
                    tags.add("application");
                    break;
                case "middleware":
                    tags.add("middleware");
                    break;
                default:
                    break;
            }
            return tags;
        }
    });

    DataStream<MetricEvent> machine = splitData.select("machine");
    DataStream<MetricEvent> docker = splitData.select("docker");
    DataStream<MetricEvent> application = splitData.select("application");
    DataStream<MetricEvent> middleware = splitData.select("middleware");

}
 

@Override
public SplitStream<T> split(OutputSelector<T> outputSelector) {
	if (requestedSideOutputs.isEmpty()) {
		wasSplitApplied = true;
		return super.split(outputSelector);
	} else {
		throw new UnsupportedOperationException("getSideOutput() and split() may not be called on the same DataStream. " +
			"As a work-around, please add a no-op map function before the split() call.");
	}
}
 

/**
 * Creates a new {@code SplitTransformation} from the given input and {@code OutputSelector}.
 *
 * @param input The input {@code Transformation}
 * @param outputSelector The output selector
 */
public SplitTransformation(
	Transformation<T> input,
		OutputSelector<T> outputSelector) {
	super("Split", input.getOutputType(), input.getParallelism());
	this.input = input;
	this.outputSelector = outputSelector;
}
 

@VisibleForTesting
public StreamNode(
		Integer id,
		@Nullable String slotSharingGroup,
		@Nullable String coLocationGroup,
		StreamOperator<?> operator,
		String operatorName,
		List<OutputSelector<?>> outputSelector,
		Class<? extends AbstractInvokable> jobVertexClass) {
	this(id, slotSharingGroup, coLocationGroup, SimpleOperatorFactory.of(operator),
			operatorName, outputSelector, jobVertexClass);
}
 

public void setOutputSelectors(List<OutputSelector<?>> outputSelectors) {
	try {
		InstantiationUtil.writeObjectToConfig(outputSelectors, this.config, OUTPUT_SELECTOR_WRAPPER);
	} catch (IOException e) {
		throw new StreamTaskException("Could not serialize output selectors", e);
	}
}
 

public <T> List<OutputSelector<T>> getOutputSelectors(ClassLoader userCodeClassloader) {
	try {
		List<OutputSelector<T>> selectors =
				InstantiationUtil.readObjectFromConfig(this.config, OUTPUT_SELECTOR_WRAPPER, userCodeClassloader);
		return selectors == null ? Collections.<OutputSelector<T>>emptyList() : selectors;

	} catch (Exception e) {
		throw new StreamTaskException("Could not read output selectors", e);
	}
}
 

public <T> void addOutputSelector(Integer vertexID, OutputSelector<T> outputSelector) {
	if (virtualPartitionNodes.containsKey(vertexID)) {
		addOutputSelector(virtualPartitionNodes.get(vertexID).f0, outputSelector);
	} else if (virtualSelectNodes.containsKey(vertexID)) {
		addOutputSelector(virtualSelectNodes.get(vertexID).f0, outputSelector);
	} else {
		getStreamNode(vertexID).addOutputSelector(outputSelector);

		if (LOG.isDebugEnabled()) {
			LOG.debug("Outputselector set for {}", vertexID);
		}
	}

}
 

@Test
public void testGetOutputs() {
	OutputSelector<Tuple1<Integer>> selector = new MyOutputSelector();
	List<String> expectedOutputs = new ArrayList<String>();
	expectedOutputs.add("0");
	expectedOutputs.add("1");
	assertEquals(expectedOutputs, selector.select(new Tuple1<Integer>(2)));
	expectedOutputs.add("2");
	assertEquals(expectedOutputs, selector.select(new Tuple1<Integer>(3)));
}
 

/**
 * Users of the test harness can call this utility method to setup the stream config
 * if there will only be a single operator to be tested. The method will setup the
 * outgoing network connection for the operator.
 *
 * <p>For more advanced test cases such as testing chains of multiple operators with the harness,
 * please manually configure the stream config.
 */
public StreamTaskMailboxTestHarnessBuilder<OUT> setupOutputForSingletonOperatorChain(
		StreamOperatorFactory<?> factory,
		OperatorID operatorID) {
	checkState(!setupCalled, "This harness was already setup.");
	setupCalled = true;
	streamConfig.setChainStart();
	streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime);
	streamConfig.setOutputSelectors(Collections.<OutputSelector<?>>emptyList());
	streamConfig.setNumberOfOutputs(1);
	streamConfig.setTypeSerializerOut(outputSerializer);
	streamConfig.setVertexID(0);

	StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() {
		private static final long serialVersionUID = 1L;
	};

	List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>();
	StreamNode sourceVertexDummy = new StreamNode(0, "group", null, dummyOperator, "source dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);
	StreamNode targetVertexDummy = new StreamNode(1, "group", null, dummyOperator, "target dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);

	outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new LinkedList<String>(), new BroadcastPartitioner<Object>(), null /* output tag */));

	streamConfig.setOutEdgesInOrder(outEdgesInOrder);
	streamConfig.setNonChainedOutputs(outEdgesInOrder);

	streamConfig.setStreamOperatorFactory(factory);
	streamConfig.setOperatorID(operatorID);

	return this;
}