org.apache.flink.api.common.state.StateDescriptor Java Examples

The following examples show how to use org.apache.flink.api.common.state.StateDescriptor. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RocksDBKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
@Nonnull
public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(
	@Nonnull TypeSerializer<N> namespaceSerializer,
	@Nonnull StateDescriptor<S, SV> stateDesc,
	@Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception {
	StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getClass());
	if (stateFactory == null) {
		String message = String.format("State %s is not supported by %s",
			stateDesc.getClass(), this.getClass());
		throw new FlinkRuntimeException(message);
	}
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult = tryRegisterKvStateInformation(
		stateDesc, namespaceSerializer, snapshotTransformFactory);
	return stateFactory.createState(stateDesc, registerResult, RocksDBKeyedStateBackend.this);
}
 
Example #2
Source File: HeapKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public <N, S extends State, T> void applyToAllKeys(
	final N namespace,
	final TypeSerializer<N> namespaceSerializer,
	final StateDescriptor<S, T> stateDescriptor,
	final KeyedStateFunction<K, S> function) throws Exception {

	try (Stream<K> keyStream = getKeys(stateDescriptor.getName(), namespace)) {

		// we copy the keys into list to avoid the concurrency problem
		// when state.clear() is invoked in function.process().
		final List<K> keys = keyStream.collect(Collectors.toList());

		final S state = getPartitionedState(
			namespace,
			namespaceSerializer,
			stateDescriptor);

		for (K key : keys) {
			setCurrentKey(key);
			function.process(key, state);
		}
	}
}
 
Example #3
Source File: MockKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <SV, SEV> StateSnapshotTransformer<SV> getStateSnapshotTransformer(
	StateDescriptor<?, SV> stateDesc,
	StateSnapshotTransformFactory<SEV> snapshotTransformFactory) {
	Optional<StateSnapshotTransformer<SEV>> original = snapshotTransformFactory.createForDeserializedState();
	if (original.isPresent()) {
		if (stateDesc instanceof ListStateDescriptor) {
			return (StateSnapshotTransformer<SV>) new StateSnapshotTransformers.ListStateSnapshotTransformer<>(original.get());
		} else if (stateDesc instanceof MapStateDescriptor) {
			return (StateSnapshotTransformer<SV>) new StateSnapshotTransformers.MapStateSnapshotTransformer<>(original.get());
		} else {
			return (StateSnapshotTransformer<SV>) original.get();
		}
	} else {
		return null;
	}
}
 
Example #4
Source File: MockKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <SV, SEV> StateSnapshotTransformer<SV> getStateSnapshotTransformer(
	StateDescriptor<?, SV> stateDesc,
	StateSnapshotTransformFactory<SEV> snapshotTransformFactory) {
	Optional<StateSnapshotTransformer<SEV>> original = snapshotTransformFactory.createForDeserializedState();
	if (original.isPresent()) {
		if (stateDesc instanceof ListStateDescriptor) {
			return (StateSnapshotTransformer<SV>) new StateSnapshotTransformers.ListStateSnapshotTransformer<>(original.get());
		} else if (stateDesc instanceof MapStateDescriptor) {
			return (StateSnapshotTransformer<SV>) new StateSnapshotTransformers.MapStateSnapshotTransformer<>(original.get());
		} else {
			return (StateSnapshotTransformer<SV>) original.get();
		}
	} else {
		return null;
	}
}
 
Example #5
Source File: AbstractKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * @see KeyedStateBackend
 */
@Override
@SuppressWarnings("unchecked")
public <N, S extends State, V> S getOrCreateKeyedState(
		final TypeSerializer<N> namespaceSerializer,
		StateDescriptor<S, V> stateDescriptor) throws Exception {
	checkNotNull(namespaceSerializer, "Namespace serializer");
	checkNotNull(keySerializer, "State key serializer has not been configured in the config. " +
			"This operation cannot use partitioned state.");

	InternalKvState<K, ?, ?> kvState = keyValueStatesByName.get(stateDescriptor.getName());
	if (kvState == null) {
		if (!stateDescriptor.isSerializerInitialized()) {
			stateDescriptor.initializeSerializerUnlessSet(executionConfig);
		}
		kvState = TtlStateFactory.createStateAndWrapWithTtlIfEnabled(
			namespaceSerializer, stateDescriptor, this, ttlTimeProvider);
		keyValueStatesByName.put(stateDescriptor.getName(), kvState);
		publishQueryableStateIfEnabled(stateDescriptor, kvState);
	}
	return (S) kvState;
}
 
Example #6
Source File: SerializationProxiesTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedStateMetaInfoSerialization() throws Exception {

	String name = "test";
	TypeSerializer<?> namespaceSerializer = LongSerializer.INSTANCE;
	TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;

	StateMetaInfoSnapshot metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
		StateDescriptor.Type.VALUE, name, namespaceSerializer, stateSerializer).snapshot();

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		StateMetaInfoSnapshotReadersWriters.getWriter().
			writeStateMetaInfoSnapshot(metaInfo, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		final StateMetaInfoReader reader = StateMetaInfoSnapshotReadersWriters.getReader(
			CURRENT_STATE_META_INFO_SNAPSHOT_VERSION, StateMetaInfoSnapshotReadersWriters.StateTypeHint.KEYED_STATE);
		metaInfo = reader.readStateMetaInfoSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
	}

	Assert.assertEquals(name, metaInfo.getName());
}
 
Example #7
Source File: MockKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <SV, SEV> StateSnapshotTransformer<SV> getStateSnapshotTransformer(
	StateDescriptor<?, SV> stateDesc,
	StateSnapshotTransformFactory<SEV> snapshotTransformFactory) {
	Optional<StateSnapshotTransformer<SEV>> original = snapshotTransformFactory.createForDeserializedState();
	if (original.isPresent()) {
		if (stateDesc instanceof ListStateDescriptor) {
			return (StateSnapshotTransformer<SV>) new StateSnapshotTransformers.ListStateSnapshotTransformer<>(original.get());
		} else if (stateDesc instanceof MapStateDescriptor) {
			return (StateSnapshotTransformer<SV>) new StateSnapshotTransformers.MapStateSnapshotTransformer<>(original.get());
		} else {
			return (StateSnapshotTransformer<SV>) original.get();
		}
	} else {
		return null;
	}
}
 
Example #8
Source File: MultiStateKeyIterator.java    From flink with Apache License 2.0 6 votes vote down vote up
/** Removes the current key from <b>ALL</b> known states in the state backend. */
@Override
public void remove() {
	if (currentKey == null) {
		return;
	}

	for (StateDescriptor<?, ?> descriptor : descriptors) {
		try {
			State state = backend.getPartitionedState(
				VoidNamespace.INSTANCE,
				VoidNamespaceSerializer.INSTANCE,
				descriptor);

			state.clear();
		} catch (Exception e) {
			throw new RuntimeException("Failed to drop partitioned state from state backend", e);
		}
	}
}
 
Example #9
Source File: EvictingWindowOperator.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public EvictingWindowOperator(WindowAssigner<? super IN, W> windowAssigner,
		TypeSerializer<W> windowSerializer,
		KeySelector<IN, K> keySelector,
		TypeSerializer<K> keySerializer,
		StateDescriptor<? extends ListState<StreamRecord<IN>>, ?> windowStateDescriptor,
		InternalWindowFunction<Iterable<IN>, OUT, K, W> windowFunction,
		Trigger<? super IN, ? super W> trigger,
		Evictor<? super IN, ? super W> evictor,
		long allowedLateness,
		OutputTag<IN> lateDataOutputTag) {

	super(windowAssigner, windowSerializer, keySelector,
		keySerializer, null, windowFunction, trigger, allowedLateness, lateDataOutputTag);

	this.evictor = checkNotNull(evictor);
	this.evictingWindowStateDescriptor = checkNotNull(windowStateDescriptor);
}
 
Example #10
Source File: HeapKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
@Nonnull
public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(
	@Nonnull TypeSerializer<N> namespaceSerializer,
	@Nonnull StateDescriptor<S, SV> stateDesc,
	@Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception {
	StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getClass());
	if (stateFactory == null) {
		String message = String.format("State %s is not supported by %s",
			stateDesc.getClass(), this.getClass());
		throw new FlinkRuntimeException(message);
	}
	StateTable<K, N, SV> stateTable = tryRegisterStateTable(
		namespaceSerializer, stateDesc, getStateSnapshotTransformFactory(stateDesc, snapshotTransformFactory));
	return stateFactory.createState(stateDesc, stateTable, getKeySerializer());
}
 
Example #11
Source File: MultiStateKeyIteratorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testIteratorRemovesFromAllDescriptors() throws Exception {
	AbstractKeyedStateBackend<Integer> keyedStateBackend = createKeyedStateBackend();

	setKey(keyedStateBackend, descriptors.get(0), 1);
	setKey(keyedStateBackend, descriptors.get(1), 1);

	MultiStateKeyIterator<Integer> iterator =
		new MultiStateKeyIterator<>(descriptors, keyedStateBackend);

	int key = iterator.next();
	Assert.assertEquals("Unexpected keys pulled from state backend", 1, key);

	iterator.remove();
	Assert.assertFalse("Failed to drop key from all descriptors in state backend", iterator.hasNext());

	for (StateDescriptor<?, ?> descriptor : descriptors) {
		Assert.assertEquals(
			"Failed to drop key for state descriptor",
			0,
			keyedStateBackend.getKeys(descriptor.getName(), VoidNamespace.INSTANCE).count());
	}
}
 
Example #12
Source File: StateDescriptorPassingTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private void validateListStateDescriptorConfigured(SingleOutputStreamOperator<?> result) {
	OneInputTransformation<?, ?> transform = (OneInputTransformation<?, ?>) result.getTransformation();
	WindowOperator<?, ?, ?, ?, ?> op = (WindowOperator<?, ?, ?, ?, ?>) transform.getOperator();
	StateDescriptor<?, ?> descr = op.getStateDescriptor();

	assertTrue(descr instanceof ListStateDescriptor);

	ListStateDescriptor<?> listDescr = (ListStateDescriptor<?>) descr;

	// this would be the first statement to fail if state descriptors were not properly initialized
	TypeSerializer<?> serializer = listDescr.getSerializer();
	assertTrue(serializer instanceof ListSerializer);

	TypeSerializer<?> elementSerializer = listDescr.getElementSerializer();
	assertTrue(elementSerializer instanceof KryoSerializer);

	Kryo kryo = ((KryoSerializer<?>) elementSerializer).getKryo();

	assertTrue("serializer registration was not properly passed on",
			kryo.getSerializer(File.class) instanceof JavaSerializer);
}
 
Example #13
Source File: AbstractQueryableStateTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
private static <K, S extends State, V> CompletableFuture<S> getKvState(
		final Deadline deadline,
		final QueryableStateClient client,
		final JobID jobId,
		final String queryName,
		final K key,
		final TypeInformation<K> keyTypeInfo,
		final StateDescriptor<S, V> stateDescriptor,
		final boolean failForUnknownKeyOrNamespace,
		final ScheduledExecutor executor) {

	final CompletableFuture<S> resultFuture = new CompletableFuture<>();
	getKvStateIgnoringCertainExceptions(
			deadline, resultFuture, client, jobId, queryName, key, keyTypeInfo,
			stateDescriptor, failForUnknownKeyOrNamespace, executor);
	return resultFuture;
}
 
Example #14
Source File: WindowOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public <S extends MergingState<?, ?>> void mergePartitionedState(
		StateDescriptor<S, ?> stateDescriptor) {
	if (mergedWindows != null && mergedWindows.size() > 0) {
		try {
			State state =
					WindowOperator.this.getOrCreateKeyedState(
							windowSerializer,
							stateDescriptor);
			if (state instanceof InternalMergingState) {
				((InternalMergingState<K, W, ?, ?, ?>) state).mergeNamespaces(window, mergedWindows);
			} else {
				throw new IllegalArgumentException(
						"The given state descriptor does not refer to a mergeable state (MergingState)");
			}
		}
		catch (Exception e) {
			throw new RuntimeException("Error while merging state.", e);
		}
	}
}
 
Example #15
Source File: ImmutableFoldingState.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public static <ACC, S extends State> S createState(
	StateDescriptor<S, ACC> stateDescriptor,
	byte[] serializedState) throws IOException {
	final ACC state = KvStateSerializer.deserializeValue(
		serializedState,
		stateDescriptor.getSerializer());
	return (S) new ImmutableFoldingState<>(state);
}
 
Example #16
Source File: AbstractKeyedStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * TODO: NOTE: This method does a lot of work caching / retrieving states just to update the namespace.
 *       This method should be removed for the sake of namespaces being lazily fetched from the keyed
 *       state backend, or being set on the state directly.
 *
 * @see KeyedStateBackend
 */
@SuppressWarnings("unchecked")
@Override
public <N, S extends State> S getPartitionedState(
		final N namespace,
		final TypeSerializer<N> namespaceSerializer,
		final StateDescriptor<S, ?> stateDescriptor) throws Exception {

	checkNotNull(namespace, "Namespace");

	if (lastName != null && lastName.equals(stateDescriptor.getName())) {
		lastState.setCurrentNamespace(namespace);
		return (S) lastState;
	}

	InternalKvState<K, ?, ?> previous = keyValueStatesByName.get(stateDescriptor.getName());
	if (previous != null) {
		lastState = previous;
		lastState.setCurrentNamespace(namespace);
		lastName = stateDescriptor.getName();
		return (S) previous;
	}

	final S state = getOrCreateKeyedState(namespaceSerializer, stateDescriptor);
	final InternalKvState<K, N, ?> kvState = (InternalKvState<K, N, ?>) state;

	lastName = stateDescriptor.getName();
	lastState = kvState;
	kvState.setCurrentNamespace(namespace);

	return state;
}
 
Example #17
Source File: WindowOperator.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new {@code WindowOperator} based on the given policies and user functions.
 */
public WindowOperator(
		WindowAssigner<? super IN, W> windowAssigner,
		TypeSerializer<W> windowSerializer,
		KeySelector<IN, K> keySelector,
		TypeSerializer<K> keySerializer,
		StateDescriptor<? extends AppendingState<IN, ACC>, ?> windowStateDescriptor,
		InternalWindowFunction<ACC, OUT, K, W> windowFunction,
		Trigger<? super IN, ? super W> trigger,
		long allowedLateness,
		OutputTag<IN> lateDataOutputTag) {

	super(windowFunction);

	checkArgument(!(windowAssigner instanceof BaseAlignedWindowAssigner),
		"The " + windowAssigner.getClass().getSimpleName() + " cannot be used with a WindowOperator. " +
			"This assigner is only used with the AccumulatingProcessingTimeWindowOperator and " +
			"the AggregatingProcessingTimeWindowOperator");

	checkArgument(allowedLateness >= 0);

	checkArgument(windowStateDescriptor == null || windowStateDescriptor.isSerializerInitialized(),
			"window state serializer is not properly initialized");

	this.windowAssigner = checkNotNull(windowAssigner);
	this.windowSerializer = checkNotNull(windowSerializer);
	this.keySelector = checkNotNull(keySelector);
	this.keySerializer = checkNotNull(keySerializer);
	this.windowStateDescriptor = windowStateDescriptor;
	this.trigger = checkNotNull(trigger);
	this.allowedLateness = allowedLateness;
	this.lateDataOutputTag = lateDataOutputTag;

	setChainingStrategy(ChainingStrategy.ALWAYS);
}
 
Example #18
Source File: ReductionsTest.java    From flink-statefun with Apache License 2.0 5 votes vote down vote up
@Nonnull
@Override
public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(
    @Nonnull TypeSerializer<N> namespaceSerializer,
    @Nonnull StateDescriptor<S, SV> stateDesc,
    @Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) {
  throw new UnsupportedOperationException();
}
 
Example #19
Source File: WindowOperator.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public <S extends State> S getPartitionedState(StateDescriptor<S, ?> stateDescriptor) {
	try {
		return WindowOperator.this.getPartitionedState(window, windowSerializer, stateDescriptor);
	} catch (Exception e) {
		throw new RuntimeException("Could not retrieve state", e);
	}
}
 
Example #20
Source File: MockInternalReducingState.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({"unchecked", "unused"})
static <N, T, S extends State, IS extends S> IS createState(
	TypeSerializer<N> namespaceSerializer,
	StateDescriptor<S, T> stateDesc) {
	ReducingStateDescriptor<T> reducingStateDesc = (ReducingStateDescriptor<T>) stateDesc;
	return (IS) new MockInternalReducingState<>(reducingStateDesc.getReduceFunction());
}
 
Example #21
Source File: RocksDBMapState.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <UK, UV, K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBMapState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		(TypeSerializer<Map<UK, UV>>) registerResult.f1.getStateSerializer(),
		(Map<UK, UV>) stateDesc.getDefaultValue(),
		backend);
}
 
Example #22
Source File: TtlStateTestBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private <S extends State> StateDescriptor<S, Object> createState() throws Exception {
	StateDescriptor<S, Object> stateDescriptor = ctx().createStateDescriptor();
	stateDescriptor.enableTimeToLive(ttlConfig);
	ctx().ttlState =
		(InternalKvState<?, String, ?>) sbetc.createState(stateDescriptor, "defaultNamespace");
	return stateDescriptor;
}
 
Example #23
Source File: AbstractQueryableStateTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
private static <K, S extends State, V> void getKvStateIgnoringCertainExceptions(
		final Deadline deadline,
		final CompletableFuture<S> resultFuture,
		final QueryableStateClient client,
		final JobID jobId,
		final String queryName,
		final K key,
		final TypeInformation<K> keyTypeInfo,
		final StateDescriptor<S, V> stateDescriptor,
		final boolean failForUnknownKeyOrNamespace,
		final ScheduledExecutor executor) {

	if (!resultFuture.isDone()) {
		CompletableFuture<S> expected = client.getKvState(jobId, queryName, key, keyTypeInfo, stateDescriptor);
		expected.whenCompleteAsync((result, throwable) -> {
			if (throwable != null) {
				if (
						throwable.getCause() instanceof CancellationException ||
						throwable.getCause() instanceof AssertionError ||
						(failForUnknownKeyOrNamespace && throwable.getCause() instanceof UnknownKeyOrNamespaceException)
				) {
					resultFuture.completeExceptionally(throwable.getCause());
				} else if (deadline.hasTimeLeft()) {
					getKvStateIgnoringCertainExceptions(
							deadline, resultFuture, client, jobId, queryName, key, keyTypeInfo,
							stateDescriptor, failForUnknownKeyOrNamespace, executor);
				}
			} else {
				resultFuture.complete(result);
			}
		}, executor);

		resultFuture.whenComplete((result, throwable) -> expected.cancel(false));
	}
}
 
Example #24
Source File: TtlStateFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
private Map<Class<? extends StateDescriptor>, SupplierWithException<IS, Exception>> createStateFactories() {
	return Stream.of(
		Tuple2.of(ValueStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createValueState),
		Tuple2.of(ListStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createListState),
		Tuple2.of(MapStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createMapState),
		Tuple2.of(ReducingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createReducingState),
		Tuple2.of(AggregatingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createAggregatingState),
		Tuple2.of(FoldingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createFoldingState)
	).collect(Collectors.toMap(t -> t.f0, t -> t.f1));
}
 
Example #25
Source File: TtlStateFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
private TtlStateFactory(
	@Nonnull TypeSerializer<N> namespaceSerializer,
	@Nonnull StateDescriptor<S, SV> stateDesc,
	@Nonnull KeyedStateBackend<K> stateBackend,
	@Nonnull TtlTimeProvider timeProvider) {
	this.namespaceSerializer = namespaceSerializer;
	this.stateDesc = stateDesc;
	this.stateBackend = stateBackend;
	this.ttlConfig = stateDesc.getTtlConfig();
	this.timeProvider = timeProvider;
	this.ttl = ttlConfig.getTtl().toMilliseconds();
	this.stateFactories = createStateFactories();
	this.incrementalCleanup = getTtlIncrementalCleanup();
}
 
Example #26
Source File: TtlStateFactory.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
private Map<Class<? extends StateDescriptor>, SupplierWithException<IS, Exception>> createStateFactories() {
	return Stream.of(
		Tuple2.of(ValueStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createValueState),
		Tuple2.of(ListStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createListState),
		Tuple2.of(MapStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createMapState),
		Tuple2.of(ReducingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createReducingState),
		Tuple2.of(AggregatingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createAggregatingState),
		Tuple2.of(FoldingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createFoldingState)
	).collect(Collectors.toMap(t -> t.f0, t -> t.f1));
}
 
Example #27
Source File: OperatorStateWriter.java    From bravo with Apache License 2.0 5 votes vote down vote up
/**
 * Defines/redefines a value state with the given name and type. This can be
 * used to create new states of an operator or change the type of an already
 * existing state.
 * <p>
 * When redefining a pre-existing state make sure you haven't added that as
 * keyed state rows before.
 * 
 * @param stateName
 * @param newState
 * @param valueSerializer
 */
public <K, V> void createNewValueState(String stateName, DataSet<Tuple2<K, V>> newState,
		TypeSerializer<V> valueSerializer) {

	metaSnapshots.put(stateName, new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, stateName,
			VoidNamespaceSerializer.INSTANCE, valueSerializer).snapshot());

	updateProxy();
	addKeyedStateRows(newState
			.map(new ValueStateToKeyedStateRow<K, V>(stateName,
					getKeySerializer(),
					valueSerializer,
					baseOpState.getMaxParallelism())));
}
 
Example #28
Source File: AbstractKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * TODO: NOTE: This method does a lot of work caching / retrieving states just to update the namespace.
 *       This method should be removed for the sake of namespaces being lazily fetched from the keyed
 *       state backend, or being set on the state directly.
 *
 * @see KeyedStateBackend
 */
@SuppressWarnings("unchecked")
@Override
public <N, S extends State> S getPartitionedState(
		final N namespace,
		final TypeSerializer<N> namespaceSerializer,
		final StateDescriptor<S, ?> stateDescriptor) throws Exception {

	checkNotNull(namespace, "Namespace");

	if (lastName != null && lastName.equals(stateDescriptor.getName())) {
		lastState.setCurrentNamespace(namespace);
		return (S) lastState;
	}

	InternalKvState<K, ?, ?> previous = keyValueStatesByName.get(stateDescriptor.getName());
	if (previous != null) {
		lastState = previous;
		lastState.setCurrentNamespace(namespace);
		lastName = stateDescriptor.getName();
		return (S) previous;
	}

	final S state = getOrCreateKeyedState(namespaceSerializer, stateDescriptor);
	final InternalKvState<K, N, ?> kvState = (InternalKvState<K, N, ?>) state;

	lastName = stateDescriptor.getName();
	lastState = kvState;
	kvState.setCurrentNamespace(namespace);

	return state;
}
 
Example #29
Source File: RocksDBMapState.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <UK, UV, K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBMapState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		(TypeSerializer<Map<UK, UV>>) registerResult.f1.getStateSerializer(),
		(Map<UK, UV>) stateDesc.getDefaultValue(),
		backend);
}
 
Example #30
Source File: ImmutableFoldingState.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public static <ACC, S extends State> S createState(
	StateDescriptor<S, ACC> stateDescriptor,
	byte[] serializedState) throws IOException {
	final ACC state = KvStateSerializer.deserializeValue(
		serializedState,
		stateDescriptor.getSerializer());
	return (S) new ImmutableFoldingState<>(state);
}