Java Code Examples for org.apache.flink.streaming.api.watermark.Watermark#getTimestamp()

@Override
public void run() {
	long lastWatermark = 0;
	while (!stopped) {
		Watermark nextWatermark = new Watermark(provider.getWatermark());
		if (lastWatermark != nextWatermark.getTimestamp()) {
			lastWatermark = nextWatermark.getTimestamp();
			ctx.emitWatermark(nextWatermark);
		}
		try {
			Thread.sleep(this.watermarkInterval);
		} catch (InterruptedException e) {
			break;
		}
	}
}
 

/**
 * Override the base implementation to completely ignore watermarks propagated from
 * upstream (we rely only on the {@link WatermarkGenerator} to emit watermarks from here).
 */
@Override
public void processWatermark(Watermark mark) throws Exception {
	// if we receive a Long.MAX_VALUE watermark we forward it since it is used
	// to signal the end of input and to not block watermark progress downstream
	if (mark.getTimestamp() == Long.MAX_VALUE && currentWatermark != Long.MAX_VALUE) {
		if (idleTimeout > 0) {
			// mark the channel active
			streamStatusMaintainer.toggleStreamStatus(StreamStatus.ACTIVE);
		}
		currentWatermark = Long.MAX_VALUE;
		output.emitWatermark(mark);
	}
}
 

@Override
public void processElement(StreamRecord<T> element) throws Exception {
	final T value = element.getValue();
	final long newTimestamp = userFunction.extractTimestamp(value,
			element.hasTimestamp() ? element.getTimestamp() : Long.MIN_VALUE);

	output.collect(element.replace(element.getValue(), newTimestamp));

	final Watermark nextWatermark = userFunction.checkAndGetNextWatermark(value, newTimestamp);
	if (nextWatermark != null && nextWatermark.getTimestamp() > currentWatermark) {
		currentWatermark = nextWatermark.getTimestamp();
		output.emitWatermark(nextWatermark);
	}
}
 

@Override
public void processWatermark(Watermark mark) throws Exception {
	// if we receive a Long.MAX_VALUE watermark we forward it since it is used
	// to signal the end of input and to not block watermark progress downstream
	if (mark.getTimestamp() == Long.MAX_VALUE && mark.getTimestamp() > currentWatermark) {
		currentWatermark = Long.MAX_VALUE;
		output.emitWatermark(mark);
	}
}
 

@Override
public void onProcessingTime(long timestamp) throws Exception {
	// register next timer
	Watermark newWatermark = userFunction.getCurrentWatermark();
	if (newWatermark != null && newWatermark.getTimestamp() > currentWatermark) {
		currentWatermark = newWatermark.getTimestamp();
		// emit watermark
		output.emitWatermark(newWatermark);
	}

	long now = getProcessingTimeService().getCurrentProcessingTime();
	getProcessingTimeService().registerTimer(now + watermarkInterval, this);
}
 

@Override
public void processWatermark(Watermark mark) throws Exception {
	wmCount++;

	if (lastWM <= mark.getTimestamp()) {
		lastWM = mark.getTimestamp();
	} else {
		throw new RuntimeException("Received watermark higher than the last one");
	}

	if (mark.getTimestamp() % 11 != 0 && mark.getTimestamp() != Long.MAX_VALUE) {
		throw new RuntimeException("Invalid watermark: " + mark.getTimestamp());
	}
}
 

public void processWatermark(Watermark mark) throws Exception {
	currentWatermark = mark.getTimestamp();
	if (inputs.isEmpty()) {
		getOneInputOperator().processWatermark(mark);
	}
	else {
		checkState(inputs.size() == 1);
		Input input = inputs.get(0);
		input.processWatermark(mark);
	}
}
 

@Test
public void testMonotonousTimestamps() {
	AssignerWithPeriodicWatermarks<String> assigner = new IngestionTimeExtractor<>();

	long maxRecordSoFar = 0L;
	long maxWatermarkSoFar = 0L;

	for (int i = 0; i < 1343; i++) {
		if (i % 7 == 1) {
			Watermark mark = assigner.getCurrentWatermark();
			assertNotNull(mark);

			// increasing watermarks
			assertTrue(mark.getTimestamp() >= maxWatermarkSoFar);
			maxWatermarkSoFar = mark.getTimestamp();

			// tight watermarks
			assertTrue(mark.getTimestamp() >= maxRecordSoFar - 1);
		} else {
			long next = assigner.extractTimestamp("a", Long.MIN_VALUE);

			// increasing timestamps
			assertTrue(next >= maxRecordSoFar);

			// timestamps are never below or at the watermark
			assertTrue(next > maxWatermarkSoFar);

			maxRecordSoFar = next;
		}

		if (i % 9 == 0) {
			try {
				Thread.sleep(1);
			} catch (InterruptedException ignored) {}
		}
	}
}
 

public void processWatermark2(Watermark mark) throws Exception {
	input2Watermark = mark.getTimestamp();
	long newMin = Math.min(input1Watermark, input2Watermark);
	if (newMin > combinedWatermark) {
		combinedWatermark = newMin;
		processWatermark(new Watermark(combinedWatermark));
	}
}
 

public void processWatermark1(Watermark mark) throws Exception {
	input1Watermark = mark.getTimestamp();
	long newMin = Math.min(input1Watermark, input2Watermark);
	if (newMin > combinedWatermark) {
		combinedWatermark = newMin;
		processWatermark(new Watermark(combinedWatermark));
	}
}
 

@Override
public int compare(Object o1, Object o2) {
	if (o1 instanceof Watermark && o2 instanceof Watermark) {
		Watermark w1 = (Watermark) o1;
		Watermark w2 = (Watermark) o2;
		return (int) (w1.getTimestamp() - w2.getTimestamp());
	} else {
		StreamRecord<WindowedValue<KV<String, Integer>>> sr0 = (StreamRecord<WindowedValue<KV<String, Integer>>>) o1;
		StreamRecord<WindowedValue<KV<String, Integer>>> sr1 = (StreamRecord<WindowedValue<KV<String, Integer>>>) o2;

		int comparison = (int) (sr0.getValue().getTimestamp().getMillis() - sr1.getValue().getTimestamp().getMillis());
		if (comparison != 0) {
			return comparison;
		}

		comparison = sr0.getValue().getValue().getKey().compareTo(sr1.getValue().getValue().getKey());
		if(comparison == 0) {
			comparison = Integer.compare(
					sr0.getValue().getValue().getValue(),
					sr1.getValue().getValue().getValue());
		}
		if(comparison == 0) {
			Collection windowsA = sr0.getValue().getWindows();
			Collection windowsB = sr1.getValue().getWindows();

			if(windowsA.size() != 1 || windowsB.size() != 1) {
				throw new IllegalStateException("A value cannot belong to more than one windows after grouping.");
			}

			BoundedWindow windowA = (BoundedWindow) windowsA.iterator().next();
			BoundedWindow windowB = (BoundedWindow) windowsB.iterator().next();
			comparison = Long.compare(windowA.maxTimestamp().getMillis(), windowB.maxTimestamp().getMillis());
		}
		return comparison;
	}
}
 

@Override
public void processElement(StreamRecord<T> element) throws Exception {
	final T value = element.getValue();
	final long newTimestamp = userFunction.extractTimestamp(value,
			element.hasTimestamp() ? element.getTimestamp() : Long.MIN_VALUE);

	output.collect(element.replace(element.getValue(), newTimestamp));

	final Watermark nextWatermark = userFunction.checkAndGetNextWatermark(value, newTimestamp);
	if (nextWatermark != null && nextWatermark.getTimestamp() > currentWatermark) {
		currentWatermark = nextWatermark.getTimestamp();
		output.emitWatermark(nextWatermark);
	}
}
 

@Override
public void processWatermark(Watermark mark) throws Exception {
	super.processWatermark(mark);
	this.currentWatermark = mark.getTimestamp();
}
 

@Override
public void processWatermark(Watermark mark) throws Exception {
	super.processWatermark(mark);
	this.currentWatermark = mark.getTimestamp();
}
 

/**
 * Called periodically to emit a watermark. Checks all shards for the current event time
 * watermark, and possibly emits the next watermark.
 *
 * <p>Shards that have not received an update for a certain interval are considered inactive so as
 * to not hold back the watermark indefinitely. When all shards are inactive, the subtask will be
 * marked as temporarily idle to not block downstream operators.
 */
@VisibleForTesting
protected void emitWatermark() {
	LOG.debug("Evaluating watermark for subtask {} time {}", indexOfThisConsumerSubtask, getCurrentTimeMillis());
	long potentialWatermark = Long.MAX_VALUE;
	long potentialNextWatermark = Long.MAX_VALUE;
	long idleTime =
		(shardIdleIntervalMillis > 0)
			? getCurrentTimeMillis() - shardIdleIntervalMillis
			: Long.MAX_VALUE;

	for (Map.Entry<Integer, ShardWatermarkState> e : shardWatermarks.entrySet()) {
		Watermark w = e.getValue().lastEmittedRecordWatermark;
		// consider only active shards, or those that would advance the watermark
		if (w != null && (e.getValue().lastUpdated >= idleTime
			|| e.getValue().emitQueue.getSize() > 0
			|| w.getTimestamp() > lastWatermark)) {
			potentialWatermark = Math.min(potentialWatermark, w.getTimestamp());
			// for sync, use the watermark of the next record, when available
			// otherwise watermark may stall when record is blocked by synchronization
			RecordEmitter.RecordQueue<RecordWrapper<T>> q = e.getValue().emitQueue;
			RecordWrapper<T> nextRecord = q.peek();
			Watermark nextWatermark = (nextRecord != null) ? nextRecord.watermark : w;
			potentialNextWatermark = Math.min(potentialNextWatermark, nextWatermark.getTimestamp());
		}
	}

	// advance watermark if possible (watermarks can only be ascending)
	if (potentialWatermark == Long.MAX_VALUE) {
		if (shardWatermarks.isEmpty() || shardIdleIntervalMillis > 0) {
			LOG.info("No active shard for subtask {}, marking the source idle.",
				indexOfThisConsumerSubtask);
			// no active shard, signal downstream operators to not wait for a watermark
			sourceContext.markAsTemporarilyIdle();
			isIdle = true;
		}
	} else {
		if (potentialWatermark > lastWatermark) {
			LOG.debug("Emitting watermark {} from subtask {}",
				potentialWatermark,
				indexOfThisConsumerSubtask);
			sourceContext.emitWatermark(new Watermark(potentialWatermark));
			lastWatermark = potentialWatermark;
			isIdle = false;
		}
		nextWatermark = potentialNextWatermark;
	}
}
 

@Override
public void processWatermark(Watermark mark) throws Exception {
	super.processWatermark(mark);
	currentWatermark = mark.getTimestamp();
}
 

@Override
public void processWatermark(Watermark mark) throws Exception {
	super.processWatermark(mark);
	currentWatermark = mark.getTimestamp();
}
 

@Override
public void processWatermark(Watermark mark) throws Exception {
	super.processWatermark(mark);
	this.currentWatermark = mark.getTimestamp();
}
 

public void processWatermark(Watermark mark) throws Exception {
	currentWatermark = mark.getTimestamp();
	oneInputOperator.processWatermark(mark);
}
 

@Override
public void processWatermark(Watermark mark) throws Exception {
	super.processWatermark(mark);
	this.currentWatermark = mark.getTimestamp();
}