org.apache.flink.api.common.state.ValueState Java Examples

private void migrateOldState() throws Exception {
	getKeyedStateBackend().applyToAllKeys(
		VoidNamespace.INSTANCE,
		VoidNamespaceSerializer.INSTANCE,
		new ValueStateDescriptor<>(
			"nfaOperatorStateName",
			new NFA.NFASerializer<>(inputSerializer)
		),
		new KeyedStateFunction<Object, ValueState<MigratedNFA<IN>>>() {
			@Override
			public void process(Object key, ValueState<MigratedNFA<IN>> state) throws Exception {
				MigratedNFA<IN> oldState = state.value();
				computationStates.update(new NFAState(oldState.getComputationStates()));
				org.apache.flink.cep.nfa.SharedBuffer<IN> sharedBuffer = oldState.getSharedBuffer();
				partialMatches.init(sharedBuffer.getEventsBuffer(), sharedBuffer.getPages());
				state.clear();
			}
		}
	);
}
 

@Override
public <S extends Serializable> ValueState<S> getKeyValueState(String name,
	Class<S> stateType,
	S defaultState) {
	checkNotNull(stateType, "The state type class must not be null");

	TypeInformation<S> typeInfo;
	try {
		typeInfo = TypeExtractor.getForClass(stateType);
	}
	catch (Exception e) {
		throw new RuntimeException("Cannot analyze type '" + stateType.getName() +
			"' from the class alone, due to generic type parameters. " +
			"Please specify the TypeInformation directly.", e);
	}

	return getKeyValueState(name, typeInfo, defaultState);
}
 

@Override
public void processElement(Integer value, Context ctx, Collector<String> out) throws Exception {
	final TimerService timerService = ctx.timerService();
	final ValueState<Integer> state = getRuntimeContext().getState(this.state);
	if (state.value() == null) {
		out.collect("INPUT:" + value);
		state.update(value);
		if (expectedTimeDomain.equals(TimeDomain.EVENT_TIME)) {
			timerService.registerEventTimeTimer(timerService.currentWatermark() + 5);
		} else {
			timerService.registerProcessingTimeTimer(timerService.currentProcessingTime() + 5);
		}
	} else {
		state.clear();
		if (expectedTimeDomain.equals(TimeDomain.EVENT_TIME)) {
			timerService.deleteEventTimeTimer(timerService.currentWatermark() + 4);
		} else {
			timerService.deleteProcessingTimeTimer(timerService.currentProcessingTime() + 4);
		}
	}
}
 

private void migrateOldState() throws Exception {
	getKeyedStateBackend().applyToAllKeys(
		VoidNamespace.INSTANCE,
		VoidNamespaceSerializer.INSTANCE,
		new ValueStateDescriptor<>(
			"nfaOperatorStateName",
			new NFA.NFASerializer<>(inputSerializer)
		),
		new KeyedStateFunction<Object, ValueState<MigratedNFA<IN>>>() {
			@Override
			public void process(Object key, ValueState<MigratedNFA<IN>> state) throws Exception {
				MigratedNFA<IN> oldState = state.value();
				computationStates.update(new NFAState(oldState.getComputationStates()));
				org.apache.flink.cep.nfa.SharedBuffer<IN> sharedBuffer = oldState.getSharedBuffer();
				partialMatches.init(sharedBuffer.getEventsBuffer(), sharedBuffer.getPages());
				state.clear();
			}
		}
	);
}
 

private void migrateOldState() throws Exception {
	getKeyedStateBackend().applyToAllKeys(
		VoidNamespace.INSTANCE,
		VoidNamespaceSerializer.INSTANCE,
		new ValueStateDescriptor<>(
			"nfaOperatorStateName",
			new NFA.NFASerializer<>(inputSerializer)
		),
		new KeyedStateFunction<Object, ValueState<MigratedNFA<IN>>>() {
			@Override
			public void process(Object key, ValueState<MigratedNFA<IN>> state) throws Exception {
				MigratedNFA<IN> oldState = state.value();
				computationStates.update(new NFAState(oldState.getComputationStates()));
				org.apache.flink.cep.nfa.SharedBuffer<IN> sharedBuffer = oldState.getSharedBuffer();
				partialMatches.init(sharedBuffer.getEventsBuffer(), sharedBuffer.getPages());
				state.clear();
			}
		}
	);
}
 

@Override
public void processElement(Integer value, Context ctx, Collector<String> out) throws Exception {
	final TimerService timerService = ctx.timerService();
	final ValueState<Integer> state = getRuntimeContext().getState(this.state);
	if (state.value() == null) {
		out.collect("INPUT:" + value);
		state.update(value);
		if (expectedTimeDomain.equals(TimeDomain.EVENT_TIME)) {
			timerService.registerEventTimeTimer(timerService.currentWatermark() + 5);
		} else {
			timerService.registerProcessingTimeTimer(timerService.currentProcessingTime() + 5);
		}
	} else {
		state.clear();
		if (expectedTimeDomain.equals(TimeDomain.EVENT_TIME)) {
			timerService.deleteEventTimeTimer(timerService.currentWatermark() + 4);
		} else {
			timerService.deleteProcessingTimeTimer(timerService.currentProcessingTime() + 4);
		}
	}
}
 

@Override
public boolean onEventTime(long time, W window) throws Exception {
	ValueState<Boolean> hasFiredState = ctx.getPartitionedState(hasFiredOnTimeStateDesc);
	Boolean hasFired = hasFiredState.value();
	if (hasFired != null && hasFired) {
		// late fire
		return lateTrigger != null && lateTrigger.onEventTime(time, window);
	} else {
		if (time == window.maxTimestamp()) {
			// fire on time and update state
			hasFiredState.update(true);
			return true;
		} else {
			// early fire
			return earlyTrigger != null && earlyTrigger.onEventTime(time, window);
		}
	}
}
 

@Override
public void processBroadcastElement(String msg, Context ctx, Collector<TaxiRide> out) throws Exception {
	DateTimeFormatter timeFormatter =
			DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss").withLocale(Locale.US).withZoneUTC();

	Long thresholdInMinutes = Long.valueOf(msg);
	Long wm = ctx.currentWatermark();
	System.out.println("QUERY: " + thresholdInMinutes + " minutes at " + timeFormatter.print(wm));

	// Collect to the output all ongoing rides that started at least thresholdInMinutes ago.
	ctx.applyToKeyedState(taxiDescriptor, new KeyedStateFunction<Long, ValueState<TaxiRide>>() {
		@Override
		public void process(Long taxiId, ValueState<TaxiRide> taxiState) throws Exception {
			TaxiRide ride = taxiState.value();
			if (ride.isStart) {
				long minutes = (wm - ride.getEventTime()) / 60000;
				if (ride.isStart && (minutes >= thresholdInMinutes)) {
					out.collect(ride);
				}
			}
		}
	});
}
 

@Override
public void processBroadcastElement(String msg, Context ctx, Collector<TaxiRide> out) throws Exception {
	DateTimeFormatter timeFormatter =
			DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss").withLocale(Locale.US).withZoneUTC();

	Long thresholdInMinutes = Long.valueOf(msg);
	Long wm = ctx.currentWatermark();
	System.out.println("QUERY: " + thresholdInMinutes + " minutes at " + timeFormatter.print(wm));

	// Collect to the output all ongoing rides that started at least thresholdInMinutes ago.
	ctx.applyToKeyedState(taxiDescriptor, new KeyedStateFunction<Long, ValueState<TaxiRide>>() {
		@Override
		public void process(Long taxiId, ValueState<TaxiRide> taxiState) throws Exception {
			throw new MissingSolutionException();
		}
	});
}
 

/**
 * Processes element to deduplicate on keys, sends current element as last row, retracts previous element if
 * needed.
 *
 * @param currentRow latest row received by deduplicate function
 * @param generateRetraction whether need to send retract message to downstream
 * @param state state of function
 * @param out underlying collector
 * @throws Exception
 */
static void processLastRow(BaseRow currentRow, boolean generateRetraction, ValueState<BaseRow> state,
		Collector<BaseRow> out) throws Exception {
	// Check message should be accumulate
	Preconditions.checkArgument(BaseRowUtil.isAccumulateMsg(currentRow));
	if (generateRetraction) {
		// state stores complete row if generateRetraction is true
		BaseRow preRow = state.value();
		state.update(currentRow);
		if (preRow != null) {
			preRow.setHeader(BaseRowUtil.RETRACT_MSG);
			out.collect(preRow);
		}
	}
	out.collect(currentRow);
}
 

@Test
public void testCopyDefaultValue() throws Exception {
	final AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	ValueStateDescriptor<IntValue> kvId = new ValueStateDescriptor<>("id", IntValue.class, new IntValue(-1));

	ValueState<IntValue> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

	backend.setCurrentKey(1);
	IntValue default1 = state.value();

	backend.setCurrentKey(2);
	IntValue default2 = state.value();

	assertNotNull(default1);
	assertNotNull(default2);
	assertEquals(default1, default2);
	assertFalse(default1 == default2);

	backend.dispose();
}
 

default void registerProcessingCleanupTimer(
		ValueState<Long> cleanupTimeState,
		long currentTime,
		long minRetentionTime,
		long maxRetentionTime,
		TimerService timerService) throws Exception {

	// last registered timer
	Long curCleanupTime = cleanupTimeState.value();

	// check if a cleanup timer is registered and
	// that the current cleanup timer won't delete state we need to keep
	if (curCleanupTime == null || (currentTime + minRetentionTime) > curCleanupTime) {
		// we need to register a new (later) timer
		long cleanupTime = currentTime + maxRetentionTime;
		// register timer and remember clean-up time
		timerService.registerProcessingTimeTimer(cleanupTime);
		// delete expired timer
		if (curCleanupTime != null) {
			timerService.deleteProcessingTimeTimer(curCleanupTime);
		}
		cleanupTimeState.update(cleanupTime);
	}
}
 

@Override
public void processElement(Integer value, Context ctx, Collector<String> out) throws Exception {
	final TimerService timerService = ctx.timerService();
	final ValueState<Integer> state = getRuntimeContext().getState(this.state);
	if (state.value() == null) {
		out.collect("INPUT:" + value);
		state.update(value);
		if (expectedTimeDomain.equals(TimeDomain.EVENT_TIME)) {
			timerService.registerEventTimeTimer(timerService.currentWatermark() + 5);
		} else {
			timerService.registerProcessingTimeTimer(timerService.currentProcessingTime() + 5);
		}
	} else {
		state.clear();
		if (expectedTimeDomain.equals(TimeDomain.EVENT_TIME)) {
			timerService.deleteEventTimeTimer(timerService.currentWatermark() + 4);
		} else {
			timerService.deleteProcessingTimeTimer(timerService.currentProcessingTime() + 4);
		}
	}
}
 

@Test
public void testCopyDefaultValue() throws Exception {
	final AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	ValueStateDescriptor<IntValue> kvId = new ValueStateDescriptor<>("id", IntValue.class, new IntValue(-1));

	ValueState<IntValue> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

	backend.setCurrentKey(1);
	IntValue default1 = state.value();

	backend.setCurrentKey(2);
	IntValue default2 = state.value();

	assertNotNull(default1);
	assertNotNull(default2);
	assertEquals(default1, default2);
	assertFalse(default1 == default2);

	backend.dispose();
}
 

@Test
public void testDisposeDeletesAllDirectories() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
	Collection<File> allFilesInDbDir =
		FileUtils.listFilesAndDirs(new File(dbPath), new AcceptAllFilter(), new AcceptAllFilter());
	try {
		ValueStateDescriptor<String> kvId =
			new ValueStateDescriptor<>("id", String.class, null);

		kvId.initializeSerializerUnlessSet(new ExecutionConfig());

		ValueState<String> state =
			backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey(1);
		state.update("Hello");

		// more than just the root directory
		assertTrue(allFilesInDbDir.size() > 1);
	} finally {
		IOUtils.closeQuietly(backend);
		backend.dispose();
	}
	allFilesInDbDir =
		FileUtils.listFilesAndDirs(new File(dbPath), new AcceptAllFilter(), new AcceptAllFilter());

	// just the root directory left
	assertEquals(1, allFilesInDbDir.size());
}
 

@Override
public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
	out.collect(value);

	ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
	if (state == null) {
		throw new RuntimeException("Missing key value state for " + value);
	}

	assertEquals(value.f1, state.value());
	getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
}
 

@Override
public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
	ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
		element.getValue().f0,
		LongSerializer.INSTANCE,
		stateDescriptor);

	assertEquals(state.value(), element.getValue().f1);
	getRuntimeContext().getAccumulator(SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR).add(1);

	output.collect(element);
}
 

/**
 * Retry a query for state for keys between 0 and {@link #maxParallelism} until
 * <tt>expected</tt> equals the value of the result tuple's second field.
 */
private void executeValueQuery(
		final Deadline deadline,
		final QueryableStateClient client,
		final JobID jobId,
		final String queryableStateName,
		final ValueStateDescriptor<Tuple2<Integer, Long>> stateDescriptor,
		final long expected) throws Exception {

	for (int key = 0; key < maxParallelism; key++) {
		boolean success = false;
		while (deadline.hasTimeLeft() && !success) {
			CompletableFuture<ValueState<Tuple2<Integer, Long>>> future = getKvState(
					deadline,
					client,
					jobId,
					queryableStateName,
					key,
					BasicTypeInfo.INT_TYPE_INFO,
					stateDescriptor,
					false,
					executor);

			Tuple2<Integer, Long> value = future.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS).value();

			assertEquals("Key mismatch", key, value.f0.intValue());
			if (expected == value.f1) {
				success = true;
			} else {
				// Retry
				Thread.sleep(RETRY_TIMEOUT);
			}
		}

		assertTrue("Did not succeed query", success);
	}
}
 

/**
 * Processes element to deduplicate on keys, sends current element if it is first row.
 *
 * @param currentRow latest row received by deduplicate function
 * @param state state of function
 * @param out underlying collector
 * @throws Exception
 */
static void processFirstRow(BaseRow currentRow, ValueState<Boolean> state, Collector<BaseRow> out)
		throws Exception {
	// Check message should be accumulate
	Preconditions.checkArgument(BaseRowUtil.isAccumulateMsg(currentRow));
	// ignore record with timestamp bigger than preRow
	if (state.value() != null) {
		return;
	}
	state.update(true);
	out.collect(currentRow);
}
 

@Override
public <S extends Serializable> ValueState<S> getKeyValueState(String name,
	TypeInformation<S> stateType,
	S defaultState) {

	checkNotNull(name, "The name of the state must not be null");
	checkNotNull(stateType, "The state type information must not be null");

	ValueStateDescriptor<S> stateDesc = new ValueStateDescriptor<>(name, stateType.createSerializer(getExecutionConfig()), defaultState);
	return getPartitionedState(stateDesc);
}
 

@Override
public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
	out.collect(value);

	ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
	if (state == null) {
		throw new RuntimeException("Missing key value state for " + value);
	}

	assertEquals(value.f1, state.value());
	getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
}
 

@Override
public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
	out.collect(value);

	ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
	if (state == null) {
		throw new RuntimeException("Missing key value state for " + value);
	}

	assertEquals(value.f1, state.value());
	getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
}
 

@Override
public <T> Accessor<T> createFlinkStateAccessor(
    FunctionType functionType, PersistedValue<T> persistedValue) {
  TypeInformation<T> typeInfo = types.registerType(persistedValue.type());
  String stateName = flinkStateName(functionType, persistedValue.name());
  ValueStateDescriptor<T> descriptor = new ValueStateDescriptor<>(stateName, typeInfo);
  configureStateTtl(descriptor, persistedValue.expiration());
  ValueState<T> handle = runtimeContext.getState(descriptor);
  return new FlinkValueAccessor<>(handle);
}
 

@Override
public void onEventTime(InternalTimer<Long, Long> timer) throws Exception {
	ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
		timer.getNamespace(),
		LongSerializer.INSTANCE,
		stateDescriptor);

	assertEquals(state.value(), timer.getNamespace());
	getRuntimeContext().getAccumulator(SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR).add(1);
}
 

@Override
public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
	out.collect(value);

	ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
	if (state == null) {
		throw new RuntimeException("Missing key value state for " + value);
	}

	assertEquals(value.f1, state.value());
	getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
}
 

@Override
public TriggerResult onElement(T element, long timestamp, W window, TriggerContext ctx) throws Exception {
	ValueState<T> lastElementState = ctx.getPartitionedState(stateDesc);
	if (lastElementState.value() == null) {
		lastElementState.update(element);
		return TriggerResult.CONTINUE;
	}
	if (deltaFunction.getDelta(lastElementState.value(), element) > this.threshold) {
		lastElementState.update(element);
		return TriggerResult.FIRE;
	}
	return TriggerResult.CONTINUE;
}
 

private void handleValue(
	Object value,
	Collector<String> out,
	TimerService timerService,
	int channel) throws IOException {
	final ValueState<String> state = getRuntimeContext().getState(this.state);
	if (state.value() == null) {
		out.collect("INPUT" + channel + ":" + value);
		state.update(String.valueOf(value));
		timerService.registerProcessingTimeTimer(timerService.currentProcessingTime() + 5);
	} else {
		state.clear();
		timerService.deleteProcessingTimeTimer(timerService.currentProcessingTime() + 4);
	}
}
 

@Override
public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
	out.collect(value);

	ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
	if (state == null) {
		throw new RuntimeException("Missing key value state for " + value);
	}

	assertEquals(value.f1, state.value());
	getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
}
 

private static <T> void touchState(ValueState<T> state, Supplier<T> elements) throws IOException {
	T elem = state.value();
	if (elem == null) {
		elem = elements.get();
	}
	state.update(elem);
}
 

@Test
public void testSetTtlTimeProvider() throws Exception {
	AbstractStreamOperator<Integer> operator = new AbstractStreamOperator<Integer>() {};
	try (AbstractStreamOperatorTestHarness<Integer> result = new AbstractStreamOperatorTestHarness<>(
			operator,
			1,
			1,
			0)) {

		result.config.setStateKeySerializer(IntSerializer.INSTANCE);

		Time timeToLive = Time.hours(1);
		result.initializeState(new OperatorSubtaskState());
		result.open();

		ValueStateDescriptor<Integer> stateDescriptor = new ValueStateDescriptor<>("test", IntSerializer.INSTANCE);
		stateDescriptor.enableTimeToLive(StateTtlConfig.newBuilder(timeToLive).build());
		KeyedStateBackend<Integer> keyedStateBackend = operator.getKeyedStateBackend();
		ValueState<Integer> state = keyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescriptor);

		int expectedValue = 42;
		keyedStateBackend.setCurrentKey(1);
		result.setStateTtlProcessingTime(0L);
		state.update(expectedValue);
		Assert.assertEquals(expectedValue, (int) state.value());
		result.setStateTtlProcessingTime(timeToLive.toMilliseconds() + 1);
		Assert.assertNull(state.value());
	}
}