Java Code Examples for org.apache.flink.api.common.state.ValueState#value()

@Test
public void testValueStateWorkWithTtl() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
	try {
		ValueStateDescriptor<MutableLong> kvId = new ValueStateDescriptor<>("id", MutableLong.class);
		kvId.enableTimeToLive(StateTtlConfig.newBuilder(Time.seconds(1)).build());

		ValueState<MutableLong> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
		backend.setCurrentKey(1);
		state.update(new MutableLong());
		state.value();
	} finally {
		backend.close();
		backend.dispose();
	}
}
 

default void registerProcessingCleanupTimer(
		ValueState<Long> cleanupTimeState,
		long currentTime,
		long minRetentionTime,
		long maxRetentionTime,
		TimerService timerService) throws Exception {

	// last registered timer
	Long curCleanupTime = cleanupTimeState.value();

	// check if a cleanup timer is registered and
	// that the current cleanup timer won't delete state we need to keep
	if (curCleanupTime == null || (currentTime + minRetentionTime) > curCleanupTime) {
		// we need to register a new (later) timer
		long cleanupTime = currentTime + maxRetentionTime;
		// register timer and remember clean-up time
		timerService.registerProcessingTimeTimer(cleanupTime);
		// delete expired timer
		if (curCleanupTime != null) {
			timerService.deleteProcessingTimeTimer(curCleanupTime);
		}
		cleanupTimeState.update(cleanupTime);
	}
}
 

@Override
public TriggerResult onElement(SensorReading r, long ts, TimeWindow w, TriggerContext ctx) throws Exception {
    // firstSeen will be false if not set yet
    ValueState<Boolean> firstSeen = ctx.getPartitionedState(
        new ValueStateDescriptor<>("firstSeen", Types.BOOLEAN));

    // register initial timer only for first element
    if (firstSeen.value() == null) {
        // compute time for next early firing by rounding watermark to second
        long t = ctx.getCurrentWatermark() + (1000 - (ctx.getCurrentWatermark() % 1000));
        ctx.registerEventTimeTimer(t);
        // register timer for the end of the window
        ctx.registerEventTimeTimer(w.getEnd());
        firstSeen.update(true);
    }
    // Continue. Do not evaluate window per element
    return TriggerResult.CONTINUE;
}
 

/**
 * Processes element to deduplicate on keys, sends current element as last row, retracts previous element if
 * needed.
 *
 * @param currentRow latest row received by deduplicate function
 * @param generateRetraction whether need to send retract message to downstream
 * @param state state of function
 * @param out underlying collector
 * @throws Exception
 */
static void processLastRow(BaseRow currentRow, boolean generateRetraction, ValueState<BaseRow> state,
		Collector<BaseRow> out) throws Exception {
	// Check message should be accumulate
	Preconditions.checkArgument(BaseRowUtil.isAccumulateMsg(currentRow));
	if (generateRetraction) {
		// state stores complete row if generateRetraction is true
		BaseRow preRow = state.value();
		state.update(currentRow);
		if (preRow != null) {
			preRow.setHeader(BaseRowUtil.RETRACT_MSG);
			out.collect(preRow);
		}
	}
	out.collect(currentRow);
}
 

@Test
public void testValueStateWorkWithTtl() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
	try {
		ValueStateDescriptor<MutableLong> kvId = new ValueStateDescriptor<>("id", MutableLong.class);
		kvId.enableTimeToLive(StateTtlConfig.newBuilder(Time.seconds(1)).build());

		ValueState<MutableLong> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
		backend.setCurrentKey(1);
		state.update(new MutableLong());
		state.value();
	} finally {
		backend.close();
		backend.dispose();
	}
}
 

private void handleValue(
	Object value,
	Collector<String> out,
	TimerService timerService,
	int channel) throws IOException {
	final ValueState<String> state = getRuntimeContext().getState(this.state);
	if (state.value() == null) {
		out.collect("INPUT" + channel + ":" + value);
		state.update(String.valueOf(value));
		timerService.registerEventTimeTimer(timerService.currentWatermark() + 5);
	} else {
		state.clear();
		timerService.deleteEventTimeTimer(timerService.currentWatermark() + 4);
	}
}
 

@Override
public TriggerResult onElement(T element, long timestamp, W window, TriggerContext ctx) throws Exception {
	ValueState<T> lastElementState = ctx.getPartitionedState(stateDesc);
	if (lastElementState.value() == null) {
		lastElementState.update(element);
		return TriggerResult.CONTINUE;
	}
	if (deltaFunction.getDelta(lastElementState.value(), element) > this.threshold) {
		lastElementState.update(element);
		return TriggerResult.FIRE;
	}
	return TriggerResult.CONTINUE;
}
 

private static <T> void touchState(ValueState<T> state, Supplier<T> elements) throws IOException {
	T elem = state.value();
	if (elem == null) {
		elem = elements.get();
	}
	state.update(elem);
}
 

/**
 * Processes element to deduplicate on keys, sends current element as last row, retracts previous element if
 * needed.
 *
 * @param currentRow latest row received by deduplicate function
 * @param generateUpdateBefore whether need to send UPDATE_BEFORE message for updates
 * @param state state of function, null if generateUpdateBefore is false
 * @param out underlying collector
 */
static void processLastRow(
		RowData currentRow,
		boolean generateUpdateBefore,
		boolean generateInsert,
		ValueState<RowData> state,
		Collector<RowData> out) throws Exception {
	// check message should be insert only.
	Preconditions.checkArgument(currentRow.getRowKind() == RowKind.INSERT);

	if (generateUpdateBefore || generateInsert) {
		// use state to keep the previous row content if we need to generate UPDATE_BEFORE
		// or use to distinguish the first row, if we need to generate INSERT
		RowData preRow = state.value();
		state.update(currentRow);
		if (preRow == null) {
			// the first row, send INSERT message
			currentRow.setRowKind(RowKind.INSERT);
			out.collect(currentRow);
		} else {
			if (generateUpdateBefore) {
				preRow.setRowKind(RowKind.UPDATE_BEFORE);
				out.collect(preRow);
			}
			currentRow.setRowKind(RowKind.UPDATE_AFTER);
			out.collect(currentRow);
		}
	} else {
		// always send UPDATE_AFTER if INSERT is not needed
		currentRow.setRowKind(RowKind.UPDATE_AFTER);
		out.collect(currentRow);
	}
}
 

/**
 * Processes element to deduplicate on keys, sends current element if it is first row.
 *
 * @param currentRow latest row received by deduplicate function
 * @param state state of function
 * @param out underlying collector
 */
static void processFirstRow(
		RowData currentRow,
		ValueState<Boolean> state,
		Collector<RowData> out) throws Exception {
	// check message should be insert only.
	Preconditions.checkArgument(currentRow.getRowKind() == RowKind.INSERT);
	// ignore record if it is not first row
	if (state.value() != null) {
		return;
	}
	state.update(true);
	// emit the first row which is INSERT message
	out.collect(currentRow);
}
 

private void handleValue(
	Object value,
	Collector<String> out,
	TimerService timerService,
	int channel) throws IOException {
	final ValueState<String> state = getRuntimeContext().getState(this.state);
	if (state.value() == null) {
		out.collect("INPUT" + channel + ":" + value);
		state.update(String.valueOf(value));
		timerService.registerEventTimeTimer(timerService.currentWatermark() + 5);
	} else {
		state.clear();
		timerService.deleteEventTimeTimer(timerService.currentWatermark() + 4);
	}
}
 

private void handleValue(
	Object value,
	Collector<String> out,
	TimerService timerService,
	int channel) throws IOException {
	final ValueState<String> state = getRuntimeContext().getState(this.state);
	if (state.value() == null) {
		out.collect("INPUT" + channel + ":" + value);
		state.update(String.valueOf(value));
		timerService.registerEventTimeTimer(timerService.currentWatermark() + 5);
	} else {
		state.clear();
		timerService.deleteEventTimeTimer(timerService.currentWatermark() + 4);
	}
}
 

@Override
public TriggerResult onElement(T element, long timestamp, W window, TriggerContext ctx) throws Exception {
	ValueState<T> lastElementState = ctx.getPartitionedState(stateDesc);
	if (lastElementState.value() == null) {
		lastElementState.update(element);
		return TriggerResult.CONTINUE;
	}
	if (deltaFunction.getDelta(lastElementState.value(), element) > this.threshold) {
		lastElementState.update(element);
		return TriggerResult.FIRE;
	}
	return TriggerResult.CONTINUE;
}
 

private void handleValue(
	Object value,
	Collector<String> out,
	TimerService timerService,
	int channel) throws IOException {
	final ValueState<String> state = getRuntimeContext().getState(this.state);
	if (state.value() == null) {
		out.collect("INPUT" + channel + ":" + value);
		state.update(String.valueOf(value));
		timerService.registerProcessingTimeTimer(timerService.currentProcessingTime() + 5);
	} else {
		state.clear();
		timerService.deleteProcessingTimeTimer(timerService.currentProcessingTime() + 4);
	}
}
 

@Test
@SuppressWarnings("unchecked")
public void testValueStateRestoreWithWrongSerializers() throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class);

		ValueState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey(1);
		state.update("1");
		backend.setCurrentKey(2);
		state.update("2");

		// draw a snapshot
		KeyedStateHandle snapshot1 = runSnapshot(
			backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);

		backend.dispose();
		// restore the first snapshot and validate it
		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
		snapshot1.discardState();

		@SuppressWarnings("unchecked")
		TypeSerializer<String> fakeStringSerializer =
			(TypeSerializer<String>) (TypeSerializer<?>) FloatSerializer.INSTANCE;

		try {
			kvId = new ValueStateDescriptor<>("id", fakeStringSerializer);

			state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

			state.value();

			fail("should recognize wrong serializers");
		} catch (StateMigrationException ignored) {
			// expected
		}
	} finally {
		backend.dispose();
	}
}
 

@Test
@SuppressWarnings("unchecked")
public void testValueStateRestoreWithWrongSerializers() throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class);

		ValueState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey(1);
		state.update("1");
		backend.setCurrentKey(2);
		state.update("2");

		// draw a snapshot
		KeyedStateHandle snapshot1 = runSnapshot(
			backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);

		backend.dispose();
		// restore the first snapshot and validate it
		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
		snapshot1.discardState();

		@SuppressWarnings("unchecked")
		TypeSerializer<String> fakeStringSerializer =
			(TypeSerializer<String>) (TypeSerializer<?>) FloatSerializer.INSTANCE;

		try {
			kvId = new ValueStateDescriptor<>("id", fakeStringSerializer);

			state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

			state.value();

			fail("should recognize wrong serializers");
		} catch (StateMigrationException ignored) {
			// expected
		}
	} finally {
		backend.dispose();
	}
}
 

@Override
String getInternal(@Nonnull ValueState<String> state) throws Exception {
	return state.value();
}
 

@Override
String getInternal(@Nonnull ValueState<String> state) throws Exception {
	return state.value();
}
 

/**
 * Verify state restore resilience when:
 *  - snapshot was taken without any Kryo registrations, specific serializers or default serializers for the state type
 *  - restored with a specific serializer for the state type
 *
 * <p> The specific serializer used on restore is {@link CustomKryoTestSerializer}, which deliberately
 * fails only on deserialization. We use the deliberate deserialization failure to acknowledge test success.
 *
 * @throws Exception expects {@link ExpectedKryoTestException} to be thrown.
 */
@Test
public void testKryoRegisteringRestoreResilienceWithRegisteredSerializer() throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();

	AbstractKeyedStateBackend<Integer> backend = null;

	try {
		backend = createKeyedBackend(IntSerializer.INSTANCE, env);

		TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);

		// make sure that we are in fact using the KryoSerializer
		assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);

		ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id", pojoType);
		ValueState<TestPojo> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		// ============== create snapshot - no Kryo registration or specific / default serializers ==============

		// make some more modifications
		backend.setCurrentKey(1);
		state.update(new TestPojo("u1", 1));

		backend.setCurrentKey(2);
		state.update(new TestPojo("u2", 2));

		KeyedStateHandle snapshot = runSnapshot(
			backend.snapshot(
				682375462378L,
				2,
				streamFactory,
				CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);

		backend.dispose();

		// ========== restore snapshot - should use specific serializer (ONLY SERIALIZATION) ==========

		env.getExecutionConfig().registerTypeWithKryoSerializer(TestPojo.class, CustomKryoTestSerializer.class);

		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot, env);

		// re-initialize to ensure that we create the KryoSerializer from scratch, otherwise
		// initializeSerializerUnlessSet would not pick up our new config
		kvId = new ValueStateDescriptor<>("id", pojoType);
		state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey(1);

		// update to test state backends that eagerly serialize, such as RocksDB
		state.update(new TestPojo("u1", 11));

		KeyedStateHandle snapshot2 = runSnapshot(
			backend.snapshot(
				682375462378L,
				2,
				streamFactory,
				CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);

		snapshot.discardState();

		backend.dispose();

		// ========= restore snapshot - should use specific serializer (FAIL ON DESERIALIZATION) =========

		env.getExecutionConfig().registerTypeWithKryoSerializer(TestPojo.class, CustomKryoTestSerializer.class);

		// on the second restore, since the custom serializer will be used for
		// deserialization, we expect the deliberate failure to be thrown
		expectedException.expect(anyOf(isA(ExpectedKryoTestException.class),
			Matchers.<Throwable>hasProperty("cause", isA(ExpectedKryoTestException.class))));

		// state backends that eagerly deserializes (such as the memory state backend) will fail here
		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot2, env);

		state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey(1);
		// state backends that lazily deserializes (such as RocksDB) will fail here
		state.value();

		backend.dispose();
	} finally {
		// ensure that native resources are also released in case of exception
		if (backend != null) {
			backend.dispose();
		}
	}
}
 

@Test
@SuppressWarnings("unchecked")
public void testValueStateRestoreWithWrongSerializers() throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class);

		ValueState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey(1);
		state.update("1");
		backend.setCurrentKey(2);
		state.update("2");

		// draw a snapshot
		KeyedStateHandle snapshot1 = runSnapshot(
			backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);

		backend.dispose();
		// restore the first snapshot and validate it
		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
		snapshot1.discardState();

		@SuppressWarnings("unchecked")
		TypeSerializer<String> fakeStringSerializer =
			(TypeSerializer<String>) (TypeSerializer<?>) FloatSerializer.INSTANCE;

		try {
			kvId = new ValueStateDescriptor<>("id", fakeStringSerializer);

			state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

			state.value();

			fail("should recognize wrong serializers");
		} catch (StateMigrationException ignored) {
			// expected
		}
	} finally {
		backend.dispose();
	}
}