org.apache.flink.streaming.runtime.streamrecord.LatencyMarker Java Examples

@Test
public void testLatencyMarker() throws Exception {
	final Map<String, Metric> metrics = new ConcurrentHashMap<>();
	final TaskMetricGroup taskMetricGroup = new StreamTaskTestHarness.TestTaskMetricGroup(metrics);

	try (StreamTaskMailboxTestHarness<String> testHarness =
			new MultipleInputStreamTaskTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO)
				.addInput(BasicTypeInfo.STRING_TYPE_INFO)
				.addInput(BasicTypeInfo.INT_TYPE_INFO)
				.addInput(BasicTypeInfo.DOUBLE_TYPE_INFO)
				.setupOutputForSingletonOperatorChain(new MapToStringMultipleInputOperatorFactory())
				.setTaskMetricGroup(taskMetricGroup)
				.build()) {
		ArrayDeque<Object> expectedOutput = new ArrayDeque<>();

		OperatorID sourceId = new OperatorID();
		LatencyMarker latencyMarker = new LatencyMarker(42L, sourceId, 0);
		testHarness.processElement(latencyMarker);
		expectedOutput.add(latencyMarker);

		assertThat(testHarness.getOutput(), contains(expectedOutput.toArray()));

		testHarness.endInput();
		testHarness.waitForTaskCompletion();
	}
}
 

private static void testLatencyStats(
	final LatencyStats.Granularity granularity,
	final Consumer<List<Tuple2<String, Histogram>>> verifier) {

	final AbstractMetricGroup<?> dummyGroup = UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup();
	final TestMetricRegistry registry = new TestMetricRegistry();
	final MetricGroup parentGroup = new GenericMetricGroup(registry, dummyGroup, PARENT_GROUP_NAME);

	final LatencyStats latencyStats = new LatencyStats(
		parentGroup,
		MetricOptions.LATENCY_HISTORY_SIZE.defaultValue(),
		OPERATOR_SUBTASK_INDEX,
		OPERATOR_ID,
		granularity);

	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 0));
	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 0));
	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 1));
	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_2, 2));
	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_2, 3));

	verifier.accept(registry.latencyHistograms);
}
 

private static void testLatencyStats(
	final LatencyStats.Granularity granularity,
	final Consumer<List<Tuple2<String, Histogram>>> verifier) {

	final AbstractMetricGroup<?> dummyGroup = UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup();
	final TestMetricRegistry registry = new TestMetricRegistry();
	final MetricGroup parentGroup = new GenericMetricGroup(registry, dummyGroup, PARENT_GROUP_NAME);

	final LatencyStats latencyStats = new LatencyStats(
		parentGroup,
		MetricOptions.LATENCY_HISTORY_SIZE.defaultValue(),
		OPERATOR_SUBTASK_INDEX,
		OPERATOR_ID,
		granularity);

	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 0));
	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 0));
	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 1));
	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_2, 2));
	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_2, 3));

	verifier.accept(registry.latencyHistograms);
}
 

public LatencyMarksEmitter(
		final ProcessingTimeService processingTimeService,
		final Output<StreamRecord<OUT>> output,
		long latencyTrackingInterval,
		final OperatorID operatorId,
		final int subtaskIndex) {

	latencyMarkTimer = processingTimeService.scheduleAtFixedRate(
		new ProcessingTimeCallback() {
			@Override
			public void onProcessingTime(long timestamp) throws Exception {
				try {
					// ProcessingTimeService callbacks are executed under the checkpointing lock
					output.emitLatencyMarker(new LatencyMarker(processingTimeService.getCurrentProcessingTime(), operatorId, subtaskIndex));
				} catch (Throwable t) {
					// we catch the Throwables here so that we don't trigger the processing
					// timer services async exception handler
					LOG.warn("Error while emitting latency marker.", t);
				}
			}
		},
		0L,
		latencyTrackingInterval);
}
 

public LatencyMarksEmitter(
		final ProcessingTimeService processingTimeService,
		final Output<StreamRecord<OUT>> output,
		long latencyTrackingInterval,
		final OperatorID operatorId,
		final int subtaskIndex) {

	latencyMarkTimer = processingTimeService.scheduleWithFixedDelay(
		new ProcessingTimeCallback() {
			@Override
			public void onProcessingTime(long timestamp) throws Exception {
				try {
					// ProcessingTimeService callbacks are executed under the checkpointing lock
					output.emitLatencyMarker(new LatencyMarker(processingTimeService.getCurrentProcessingTime(), operatorId, subtaskIndex));
				} catch (Throwable t) {
					// we catch the Throwables here so that we don't trigger the processing
					// timer services async exception handler
					LOG.warn("Error while emitting latency marker.", t);
				}
			}
		},
		0L,
		latencyTrackingInterval);
}
 

@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
	serializationDelegate.setInstance(latencyMarker);

	try {
		recordWriter.randomEmit(serializationDelegate);
	}
	catch (Exception e) {
		throw new RuntimeException(e.getMessage(), e);
	}
}
 

@Override
protected void reportOrForwardLatencyMarker(LatencyMarker marker) {
	// all operators are tracking latencies
	this.latencyStats.reportLatency(marker);

	// sinks don't forward latency markers
}
 

@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
	if (outputs.length <= 0) {
		// ignore
	} else if (outputs.length == 1) {
		outputs[0].emitLatencyMarker(latencyMarker);
	} else {
		// randomly select an output
		outputs[random.nextInt(outputs.length)].emitLatencyMarker(latencyMarker);
	}
}
 

protected void reportOrForwardLatencyMarker(LatencyMarker marker) {
	// all operators are tracking latencies
	this.latencyStats.reportLatency(marker);

	// everything except sinks forwards latency markers
	this.output.emitLatencyMarker(marker);
}
 

@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
	serializationDelegate.setInstance(latencyMarker);

	try {
		recordWriter.randomEmit(serializationDelegate);
	}
	catch (Exception e) {
		throw new RuntimeException(e.getMessage(), e);
	}
}
 

@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
	try {
		operator.processLatencyMarker(latencyMarker);
	}
	catch (Exception e) {
		throw new ExceptionInChainedOperatorException(e);
	}
}
 

@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base
		.addGroup("source_id", String.valueOf(marker.getOperatorId()))
		.addGroup("source_subtask_index", String.valueOf(marker.getSubtaskIndex()));
}
 

@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
	try {
		operator.processLatencyMarker(latencyMarker);
	}
	catch (Exception e) {
		throw new ExceptionInChainedOperatorException(e);
	}
}
 

@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base
		.addGroup("source_id", String.valueOf(marker.getOperatorId()))
		.addGroup("source_subtask_index", String.valueOf(marker.getSubtaskIndex()));
}
 

@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
	serializationDelegate.setInstance(latencyMarker);

	try {
		recordWriter.randomEmit(serializationDelegate);
	}
	catch (Exception e) {
		throw new RuntimeException(e.getMessage(), e);
	}
}
 

@Override
protected void reportOrForwardLatencyMarker(LatencyMarker marker) {
	// all operators are tracking latencies
	this.latencyStats.reportLatency(marker);

	// sinks don't forward latency markers
}
 

@Override
protected void reportOrForwardLatencyMarker(LatencyMarker marker) {
	// all operators are tracking latencies
	this.latencyStats.reportLatency(marker);

	// sinks don't forward latency markers
}
 

@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
	if (outputs.length <= 0) {
		// ignore
	} else if (outputs.length == 1) {
		outputs[0].emitLatencyMarker(latencyMarker);
	} else {
		// randomly select an output
		outputs[random.nextInt(outputs.length)].emitLatencyMarker(latencyMarker);
	}
}
 

@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
	try {
		operator.processLatencyMarker(latencyMarker);
	}
	catch (Exception e) {
		throw new ExceptionInChainedOperatorException(e);
	}
}
 

@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base
		.addGroup("source_id", String.valueOf(marker.getOperatorId()));
}
 

@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base
		.addGroup("source_id", String.valueOf(marker.getOperatorId()))
		.addGroup("source_subtask_index", String.valueOf(marker.getSubtaskIndex()));
}
 

@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base;
}
 

@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base
		.addGroup("source_id", String.valueOf(marker.getOperatorId()));
}
 

@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base;
}
 

@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) throws Exception {
	if (inputIndex == 0) {
		operator.processLatencyMarker1(latencyMarker);
	} else {
		operator.processLatencyMarker2(latencyMarker);
	}
}
 

@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base
		.addGroup("source_id", String.valueOf(marker.getOperatorId()));
}
 

@Override
protected void reportOrForwardLatencyMarker(LatencyMarker marker) {
	// all operators are tracking latencies
	this.latencyStats.reportLatency(marker);

	// sinks don't forward latency markers
}
 

protected void reportOrForwardLatencyMarker(LatencyMarker marker) {
	// all operators are tracking latencies
	this.latencyStats.reportLatency(marker);

	// everything except sinks forwards latency markers
	this.output.emitLatencyMarker(marker);
}
 

@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
}
 

@Override
public void processLatencyMarker2(LatencyMarker latencyMarker) throws Exception {
	// ignore
}