Java Code Examples for org.apache.flink.api.common.state.MapStateDescriptor#initializeSerializerUnlessSet()

@Override
public <UK, UV> MapState<UK, UV> getMapState(MapStateDescriptor<UK, UV> stateProperties) {
	requireNonNull(stateProperties, "The state properties must not be null");
	try {
		stateProperties.initializeSerializerUnlessSet(executionConfig);
		MapState<UK, UV> originalState = getPartitionedState(stateProperties);
		return new UserFacingMapState<>(originalState);
	} catch (Exception e) {
		throw new RuntimeException("Error while getting state", e);
	}
}
 

@Override
public <K, V> ReadOnlyBroadcastState<K, V> getBroadcastState(MapStateDescriptor<K, V> stateDescriptor) {
	Preconditions.checkNotNull(stateDescriptor);

	stateDescriptor.initializeSerializerUnlessSet(config);
	ReadOnlyBroadcastState<K, V> state = (ReadOnlyBroadcastState<K, V>) states.get(stateDescriptor);
	if (state == null) {
		throw new IllegalArgumentException("The requested state does not exist. " +
				"Check for typos in your state descriptor, or specify the state descriptor " +
				"in the datastream.broadcast(...) call if you forgot to register it.");
	}
	return state;
}
 

@Override
public <K, V> ReadOnlyBroadcastState<K, V> getBroadcastState(MapStateDescriptor<K, V> stateDescriptor) {
	Preconditions.checkNotNull(stateDescriptor);

	stateDescriptor.initializeSerializerUnlessSet(config);
	ReadOnlyBroadcastState<K, V> state = (ReadOnlyBroadcastState<K, V>) states.get(stateDescriptor);
	if (state == null) {
		throw new IllegalArgumentException("The requested state does not exist. " +
				"Check for typos in your state descriptor, or specify the state descriptor " +
				"in the datastream.broadcast(...) call if you forgot to register it.");
	}
	return state;
}
 

@Override
public <K, V> BroadcastState<K, V> getBroadcastState(MapStateDescriptor<K, V> stateDescriptor) {
	Preconditions.checkNotNull(stateDescriptor);

	stateDescriptor.initializeSerializerUnlessSet(config);
	BroadcastState<K, V> state = (BroadcastState<K, V>) states.get(stateDescriptor);
	if (state == null) {
		throw new IllegalArgumentException("The requested state does not exist. " +
				"Check for typos in your state descriptor, or specify the state descriptor " +
				"in the datastream.broadcast(...) call if you forgot to register it.");
	}
	return state;
}
 

@Override
public <K, V> ReadOnlyBroadcastState<K, V> getBroadcastState(MapStateDescriptor<K, V> stateDescriptor) {
	Preconditions.checkNotNull(stateDescriptor);

	stateDescriptor.initializeSerializerUnlessSet(config);
	ReadOnlyBroadcastState<K, V> state = (ReadOnlyBroadcastState<K, V>) states.get(stateDescriptor);
	if (state == null) {
		throw new IllegalArgumentException("The requested state does not exist. " +
				"Check for typos in your state descriptor, or specify the state descriptor " +
				"in the datastream.broadcast(...) call if you forgot to register it.");
	}
	return state;
}
 

@Override
public <K, V> ReadOnlyBroadcastState<K, V> getBroadcastState(MapStateDescriptor<K, V> stateDescriptor) {
	Preconditions.checkNotNull(stateDescriptor);

	stateDescriptor.initializeSerializerUnlessSet(config);
	ReadOnlyBroadcastState<K, V> state = (ReadOnlyBroadcastState<K, V>) states.get(stateDescriptor);
	if (state == null) {
		throw new IllegalArgumentException("The requested state does not exist. " +
				"Check for typos in your state descriptor, or specify the state descriptor " +
				"in the datastream.broadcast(...) call if you forgot to register it.");
	}
	return state;
}
 

@Override
public  <K, V> ReadOnlyBroadcastState<K, V> getBroadcastState(MapStateDescriptor<K, V> stateDescriptor) {
	Preconditions.checkNotNull(stateDescriptor);

	stateDescriptor.initializeSerializerUnlessSet(config);
	ReadOnlyBroadcastState<K, V> state = (ReadOnlyBroadcastState<K, V>) states.get(stateDescriptor);
	if (state == null) {
		throw new IllegalArgumentException("The requested state does not exist. " +
				"Check for typos in your state descriptor, or specify the state descriptor " +
				"in the datastream.broadcast(...) call if you forgot to register it.");
	}
	return state;
}
 

@Override
public <K, V> ReadOnlyBroadcastState<K, V> getBroadcastState(MapStateDescriptor<K, V> stateDescriptor) {
	Preconditions.checkNotNull(stateDescriptor);

	stateDescriptor.initializeSerializerUnlessSet(config);
	ReadOnlyBroadcastState<K, V> state = (ReadOnlyBroadcastState<K, V>) states.get(stateDescriptor);
	if (state == null) {
		throw new IllegalArgumentException("The requested state does not exist. " +
				"Check for typos in your state descriptor, or specify the state descriptor " +
				"in the datastream.broadcast(...) call if you forgot to register it.");
	}
	return state;
}
 

@Override
public  <K, V> ReadOnlyBroadcastState<K, V> getBroadcastState(MapStateDescriptor<K, V> stateDescriptor) {
	Preconditions.checkNotNull(stateDescriptor);

	stateDescriptor.initializeSerializerUnlessSet(config);
	ReadOnlyBroadcastState<K, V> state = (ReadOnlyBroadcastState<K, V>) states.get(stateDescriptor);
	if (state == null) {
		throw new IllegalArgumentException("The requested state does not exist. " +
				"Check for typos in your state descriptor, or specify the state descriptor " +
				"in the datastream.broadcast(...) call if you forgot to register it.");
	}
	return state;
}
 

@Override
public <K, V> BroadcastState<K, V> getBroadcastState(MapStateDescriptor<K, V> stateDescriptor) {
	Preconditions.checkNotNull(stateDescriptor);

	stateDescriptor.initializeSerializerUnlessSet(config);
	BroadcastState<K, V> state = (BroadcastState<K, V>) states.get(stateDescriptor);
	if (state == null) {
		throw new IllegalArgumentException("The requested state does not exist. " +
				"Check for typos in your state descriptor, or specify the state descriptor " +
				"in the datastream.broadcast(...) call if you forgot to register it.");
	}
	return state;
}
 

@Override
public <K, V> ReadOnlyBroadcastState<K, V> getBroadcastState(MapStateDescriptor<K, V> stateDescriptor) {
	Preconditions.checkNotNull(stateDescriptor);

	stateDescriptor.initializeSerializerUnlessSet(config);
	ReadOnlyBroadcastState<K, V> state = (ReadOnlyBroadcastState<K, V>) states.get(stateDescriptor);
	if (state == null) {
		throw new IllegalArgumentException("The requested state does not exist. " +
				"Check for typos in your state descriptor, or specify the state descriptor " +
				"in the datastream.broadcast(...) call if you forgot to register it.");
	}
	return state;
}
 

@Override
public <K, V> BroadcastState<K, V> getBroadcastState(MapStateDescriptor<K, V> stateDescriptor) {
	Preconditions.checkNotNull(stateDescriptor);

	stateDescriptor.initializeSerializerUnlessSet(config);
	BroadcastState<K, V> state = (BroadcastState<K, V>) states.get(stateDescriptor);
	if (state == null) {
		throw new IllegalArgumentException("The requested state does not exist. " +
				"Check for typos in your state descriptor, or specify the state descriptor " +
				"in the datastream.broadcast(...) call if you forgot to register it.");
	}
	return state;
}
 

@Override
public <K, V> BroadcastState<K, V> getBroadcastState(MapStateDescriptor<K, V> stateDescriptor) {
	Preconditions.checkNotNull(stateDescriptor);

	stateDescriptor.initializeSerializerUnlessSet(config);
	BroadcastState<K, V> state = (BroadcastState<K, V>) states.get(stateDescriptor);
	if (state == null) {
		throw new IllegalArgumentException("The requested state does not exist. " +
				"Check for typos in your state descriptor, or specify the state descriptor " +
				"in the datastream.broadcast(...) call if you forgot to register it.");
	}
	return state;
}
 

@Override
public <UK, UV> MapState<UK, UV> getMapState(MapStateDescriptor<UK, UV> stateProperties) {
	requireNonNull(stateProperties, "The state properties must not be null");
	try {
		stateProperties.initializeSerializerUnlessSet(executionConfig);
		MapState<UK, UV> originalState = getPartitionedState(stateProperties);
		return new UserFacingMapState<>(originalState);
	} catch (Exception e) {
		throw new RuntimeException("Error while getting state", e);
	}
}
 

@Override
public <UK, UV> MapState<UK, UV> getMapState(MapStateDescriptor<UK, UV> stateProperties) {
	KeyedStateStore keyedStateStore = checkPreconditionsAndGetKeyedStateStore(stateProperties);
	stateProperties.initializeSerializerUnlessSet(getExecutionConfig());
	return keyedStateStore.getMapState(stateProperties);
}
 

@SuppressWarnings("unchecked")
@Override
public <K, V> BroadcastState<K, V> getBroadcastState(final MapStateDescriptor<K, V> stateDescriptor) throws StateMigrationException {

	Preconditions.checkNotNull(stateDescriptor);
	String name = Preconditions.checkNotNull(stateDescriptor.getName());

	BackendWritableBroadcastState<K, V> previous =
		(BackendWritableBroadcastState<K, V>) accessedBroadcastStatesByName.get(name);

	if (previous != null) {
		checkStateNameAndMode(
				previous.getStateMetaInfo().getName(),
				name,
				previous.getStateMetaInfo().getAssignmentMode(),
				OperatorStateHandle.Mode.BROADCAST);
		return previous;
	}

	stateDescriptor.initializeSerializerUnlessSet(getExecutionConfig());
	TypeSerializer<K> broadcastStateKeySerializer = Preconditions.checkNotNull(stateDescriptor.getKeySerializer());
	TypeSerializer<V> broadcastStateValueSerializer = Preconditions.checkNotNull(stateDescriptor.getValueSerializer());

	BackendWritableBroadcastState<K, V> broadcastState =
		(BackendWritableBroadcastState<K, V>) registeredBroadcastStates.get(name);

	if (broadcastState == null) {
		broadcastState = new HeapBroadcastState<>(
				new RegisteredBroadcastStateBackendMetaInfo<>(
						name,
						OperatorStateHandle.Mode.BROADCAST,
						broadcastStateKeySerializer,
						broadcastStateValueSerializer));
		registeredBroadcastStates.put(name, broadcastState);
	} else {
		// has restored state; check compatibility of new state access

		checkStateNameAndMode(
				broadcastState.getStateMetaInfo().getName(),
				name,
				broadcastState.getStateMetaInfo().getAssignmentMode(),
				OperatorStateHandle.Mode.BROADCAST);

		RegisteredBroadcastStateBackendMetaInfo<K, V> restoredBroadcastStateMetaInfo = broadcastState.getStateMetaInfo();

		// check whether new serializers are incompatible
		TypeSerializerSchemaCompatibility<K> keyCompatibility =
			restoredBroadcastStateMetaInfo.updateKeySerializer(broadcastStateKeySerializer);
		if (keyCompatibility.isIncompatible()) {
			throw new StateMigrationException("The new key typeSerializer for broadcast state must not be incompatible.");
		}

		TypeSerializerSchemaCompatibility<V> valueCompatibility =
			restoredBroadcastStateMetaInfo.updateValueSerializer(broadcastStateValueSerializer);
		if (valueCompatibility.isIncompatible()) {
			throw new StateMigrationException("The new value typeSerializer for broadcast state must not be incompatible.");
		}

		broadcastState.setStateMetaInfo(restoredBroadcastStateMetaInfo);
	}

	accessedBroadcastStatesByName.put(name, broadcastState);
	return broadcastState;
}
 

@Test
public void testMapStateMigrationAfterHashMapSerRemoval() throws Exception {
	ClassLoader cl = getClass().getClassLoader();
	URL resource = cl.getResource("heap_keyed_statebackend_1_5_map.snapshot");

	Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");

	final SnapshotResult<KeyedStateHandle> stateHandles;
	try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
		stateHandles = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
	}
	final KeyedStateHandle stateHandle = stateHandles.getJobManagerOwnedSnapshot();
	try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
		final Integer namespace1 = 1;
		final Integer namespace2 = 2;
		final Integer namespace3 = 3;

		final MapStateDescriptor<Long, Long> stateDescr = new MapStateDescriptor<>("my-map-state", Long.class, Long.class);
		stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());

		InternalMapState<String, Integer, Long, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);

		keyedBackend.setCurrentKey("abc");
		state.setCurrentNamespace(namespace1);
		assertEquals(33L, (long) state.get(33L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace2);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(11L, (long) state.get(11L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(44L, (long) state.get(44L));
		assertEquals(1, getStateSize(state));

		keyedBackend.setCurrentKey("def");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(3, getStateSize(state));

		keyedBackend.setCurrentKey("jkl");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		keyedBackend.setCurrentKey("mno");
		state.setCurrentNamespace(namespace3);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = keyedBackend.snapshot(
				1L,
				1L,
				new MemCheckpointStreamFactory(4 * 1024 * 1024),
				CheckpointOptions.forCheckpointWithDefaultLocation());

		snapshot.run();
	}
}
 

@Override
public <UK, UV> MapState<UK, UV> getMapState(MapStateDescriptor<UK, UV> stateProperties) {
	KeyedStateStore keyedStateStore = checkPreconditionsAndGetKeyedStateStore(stateProperties);
	stateProperties.initializeSerializerUnlessSet(getExecutionConfig());
	return keyedStateStore.getMapState(stateProperties);
}
 

@Override
public <UK, UV> MapState<UK, UV> getMapState(MapStateDescriptor<UK, UV> stateProperties) {
	KeyedStateStore keyedStateStore = checkPreconditionsAndGetKeyedStateStore(stateProperties);
	stateProperties.initializeSerializerUnlessSet(getExecutionConfig());
	return keyedStateStore.getMapState(stateProperties);
}
 

@Test
public void testMapStateMigrationAfterHashMapSerRemoval() throws Exception {
	ClassLoader cl = getClass().getClassLoader();
	URL resource = cl.getResource("heap_keyed_statebackend_1_5_map.snapshot");

	Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");

	final SnapshotResult<KeyedStateHandle> stateHandles;
	try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
		stateHandles = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
	}
	final KeyedStateHandle stateHandle = stateHandles.getJobManagerOwnedSnapshot();
	try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
		final Integer namespace1 = 1;
		final Integer namespace2 = 2;
		final Integer namespace3 = 3;

		final MapStateDescriptor<Long, Long> stateDescr = new MapStateDescriptor<>("my-map-state", Long.class, Long.class);
		stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());

		InternalMapState<String, Integer, Long, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);

		keyedBackend.setCurrentKey("abc");
		state.setCurrentNamespace(namespace1);
		assertEquals(33L, (long) state.get(33L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace2);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(11L, (long) state.get(11L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(44L, (long) state.get(44L));
		assertEquals(1, getStateSize(state));

		keyedBackend.setCurrentKey("def");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(3, getStateSize(state));

		keyedBackend.setCurrentKey("jkl");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		keyedBackend.setCurrentKey("mno");
		state.setCurrentNamespace(namespace3);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = keyedBackend.snapshot(
				1L,
				1L,
				new MemCheckpointStreamFactory(4 * 1024 * 1024),
				CheckpointOptions.forCheckpointWithDefaultLocation());

		snapshot.run();
	}
}