Java Code Examples for org.apache.flink.runtime.state.internal.InternalMapState#setCurrentNamespace()
The following examples show how to use
org.apache.flink.runtime.state.internal.InternalMapState#setCurrentNamespace() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: HeapKeyedStateBackendSnapshotMigrationTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
public void testMapStateMigrationAfterHashMapSerRemoval() throws Exception {
ClassLoader cl = getClass().getClassLoader();
URL resource = cl.getResource("heap_keyed_statebackend_1_5_map.snapshot");
Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");
final SnapshotResult<KeyedStateHandle> stateHandles;
try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
stateHandles = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
}
final KeyedStateHandle stateHandle = stateHandles.getJobManagerOwnedSnapshot();
try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
final Integer namespace1 = 1;
final Integer namespace2 = 2;
final Integer namespace3 = 3;
final MapStateDescriptor<Long, Long> stateDescr = new MapStateDescriptor<>("my-map-state", Long.class, Long.class);
stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());
InternalMapState<String, Integer, Long, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);
keyedBackend.setCurrentKey("abc");
state.setCurrentNamespace(namespace1);
assertEquals(33L, (long) state.get(33L));
assertEquals(55L, (long) state.get(55L));
assertEquals(2, getStateSize(state));
state.setCurrentNamespace(namespace2);
assertEquals(22L, (long) state.get(22L));
assertEquals(11L, (long) state.get(11L));
assertEquals(2, getStateSize(state));
state.setCurrentNamespace(namespace3);
assertEquals(44L, (long) state.get(44L));
assertEquals(1, getStateSize(state));
keyedBackend.setCurrentKey("def");
state.setCurrentNamespace(namespace1);
assertEquals(11L, (long) state.get(11L));
assertEquals(44L, (long) state.get(44L));
assertEquals(2, getStateSize(state));
state.setCurrentNamespace(namespace3);
assertEquals(22L, (long) state.get(22L));
assertEquals(55L, (long) state.get(55L));
assertEquals(33L, (long) state.get(33L));
assertEquals(3, getStateSize(state));
keyedBackend.setCurrentKey("jkl");
state.setCurrentNamespace(namespace1);
assertEquals(11L, (long) state.get(11L));
assertEquals(22L, (long) state.get(22L));
assertEquals(33L, (long) state.get(33L));
assertEquals(44L, (long) state.get(44L));
assertEquals(55L, (long) state.get(55L));
assertEquals(5, getStateSize(state));
keyedBackend.setCurrentKey("mno");
state.setCurrentNamespace(namespace3);
assertEquals(11L, (long) state.get(11L));
assertEquals(22L, (long) state.get(22L));
assertEquals(33L, (long) state.get(33L));
assertEquals(44L, (long) state.get(44L));
assertEquals(55L, (long) state.get(55L));
assertEquals(5, getStateSize(state));
RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = keyedBackend.snapshot(
1L,
1L,
new MemCheckpointStreamFactory(4 * 1024 * 1024),
CheckpointOptions.forCheckpointWithDefaultLocation());
snapshot.run();
}
}
Example 2
| Source File: KvStateRequestSerializerTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
/**
* Verifies that the serialization of a map using the given map state
* matches the deserialization with {@link KvStateSerializer#deserializeList}.
*
* @param key
* key of the map state
* @param mapState
* map state using the {@link VoidNamespace}, must also be a {@link InternalKvState} instance
*
* @throws Exception
*/
public static void testMapSerialization(
final long key,
final InternalMapState<Long, VoidNamespace, Long, String> mapState) throws Exception {
TypeSerializer<Long> userKeySerializer = LongSerializer.INSTANCE;
TypeSerializer<String> userValueSerializer = StringSerializer.INSTANCE;
mapState.setCurrentNamespace(VoidNamespace.INSTANCE);
// Map
final int numElements = 10;
final Map<Long, String> expectedValues = new HashMap<>();
for (int i = 1; i <= numElements; i++) {
final long value = ThreadLocalRandom.current().nextLong();
expectedValues.put(value, Long.toString(value));
mapState.put(value, Long.toString(value));
}
expectedValues.put(0L, null);
mapState.put(0L, null);
final byte[] serializedKey =
KvStateSerializer.serializeKeyAndNamespace(
key, LongSerializer.INSTANCE,
VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
final byte[] serializedValues = mapState.getSerializedValue(
serializedKey,
mapState.getKeySerializer(),
mapState.getNamespaceSerializer(),
mapState.getValueSerializer());
Map<Long, String> actualValues = KvStateSerializer.deserializeMap(serializedValues, userKeySerializer, userValueSerializer);
assertEquals(expectedValues.size(), actualValues.size());
for (Map.Entry<Long, String> actualEntry : actualValues.entrySet()) {
assertEquals(expectedValues.get(actualEntry.getKey()), actualEntry.getValue());
}
// Single value
ByteArrayOutputStream baos = new ByteArrayOutputStream();
long expectedKey = ThreadLocalRandom.current().nextLong();
String expectedValue = Long.toString(expectedKey);
byte[] isNull = {0};
baos.write(KvStateSerializer.serializeValue(expectedKey, userKeySerializer));
baos.write(isNull);
baos.write(KvStateSerializer.serializeValue(expectedValue, userValueSerializer));
byte[] serializedValue = baos.toByteArray();
Map<Long, String> actualValue = KvStateSerializer.deserializeMap(serializedValue, userKeySerializer, userValueSerializer);
assertEquals(1, actualValue.size());
assertEquals(expectedValue, actualValue.get(expectedKey));
}
Example 3
| Source File: HeapKeyedStateBackendSnapshotMigrationTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testMapStateMigrationAfterHashMapSerRemoval() throws Exception {
ClassLoader cl = getClass().getClassLoader();
URL resource = cl.getResource("heap_keyed_statebackend_1_5_map.snapshot");
Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");
final SnapshotResult<KeyedStateHandle> stateHandles;
try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
stateHandles = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
}
final KeyedStateHandle stateHandle = stateHandles.getJobManagerOwnedSnapshot();
try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
final Integer namespace1 = 1;
final Integer namespace2 = 2;
final Integer namespace3 = 3;
final MapStateDescriptor<Long, Long> stateDescr = new MapStateDescriptor<>("my-map-state", Long.class, Long.class);
stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());
InternalMapState<String, Integer, Long, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);
keyedBackend.setCurrentKey("abc");
state.setCurrentNamespace(namespace1);
assertEquals(33L, (long) state.get(33L));
assertEquals(55L, (long) state.get(55L));
assertEquals(2, getStateSize(state));
state.setCurrentNamespace(namespace2);
assertEquals(22L, (long) state.get(22L));
assertEquals(11L, (long) state.get(11L));
assertEquals(2, getStateSize(state));
state.setCurrentNamespace(namespace3);
assertEquals(44L, (long) state.get(44L));
assertEquals(1, getStateSize(state));
keyedBackend.setCurrentKey("def");
state.setCurrentNamespace(namespace1);
assertEquals(11L, (long) state.get(11L));
assertEquals(44L, (long) state.get(44L));
assertEquals(2, getStateSize(state));
state.setCurrentNamespace(namespace3);
assertEquals(22L, (long) state.get(22L));
assertEquals(55L, (long) state.get(55L));
assertEquals(33L, (long) state.get(33L));
assertEquals(3, getStateSize(state));
keyedBackend.setCurrentKey("jkl");
state.setCurrentNamespace(namespace1);
assertEquals(11L, (long) state.get(11L));
assertEquals(22L, (long) state.get(22L));
assertEquals(33L, (long) state.get(33L));
assertEquals(44L, (long) state.get(44L));
assertEquals(55L, (long) state.get(55L));
assertEquals(5, getStateSize(state));
keyedBackend.setCurrentKey("mno");
state.setCurrentNamespace(namespace3);
assertEquals(11L, (long) state.get(11L));
assertEquals(22L, (long) state.get(22L));
assertEquals(33L, (long) state.get(33L));
assertEquals(44L, (long) state.get(44L));
assertEquals(55L, (long) state.get(55L));
assertEquals(5, getStateSize(state));
RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = keyedBackend.snapshot(
1L,
1L,
new MemCheckpointStreamFactory(4 * 1024 * 1024),
CheckpointOptions.forCheckpointWithDefaultLocation());
snapshot.run();
}
}
Example 4
| Source File: KvStateRequestSerializerTest.java From flink with Apache License 2.0 | 4 votes |
|
/**
* Verifies that the serialization of a map using the given map state
* matches the deserialization with {@link KvStateSerializer#deserializeList}.
*
* @param key
* key of the map state
* @param mapState
* map state using the {@link VoidNamespace}, must also be a {@link InternalKvState} instance
*
* @throws Exception
*/
public static void testMapSerialization(
final long key,
final InternalMapState<Long, VoidNamespace, Long, String> mapState) throws Exception {
TypeSerializer<Long> userKeySerializer = LongSerializer.INSTANCE;
TypeSerializer<String> userValueSerializer = StringSerializer.INSTANCE;
mapState.setCurrentNamespace(VoidNamespace.INSTANCE);
// Map
final int numElements = 10;
final Map<Long, String> expectedValues = new HashMap<>();
for (int i = 1; i <= numElements; i++) {
final long value = ThreadLocalRandom.current().nextLong();
expectedValues.put(value, Long.toString(value));
mapState.put(value, Long.toString(value));
}
expectedValues.put(0L, null);
mapState.put(0L, null);
final byte[] serializedKey =
KvStateSerializer.serializeKeyAndNamespace(
key, LongSerializer.INSTANCE,
VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
final byte[] serializedValues = mapState.getSerializedValue(
serializedKey,
mapState.getKeySerializer(),
mapState.getNamespaceSerializer(),
mapState.getValueSerializer());
Map<Long, String> actualValues = KvStateSerializer.deserializeMap(serializedValues, userKeySerializer, userValueSerializer);
assertEquals(expectedValues.size(), actualValues.size());
for (Map.Entry<Long, String> actualEntry : actualValues.entrySet()) {
assertEquals(expectedValues.get(actualEntry.getKey()), actualEntry.getValue());
}
// Single value
ByteArrayOutputStream baos = new ByteArrayOutputStream();
long expectedKey = ThreadLocalRandom.current().nextLong();
String expectedValue = Long.toString(expectedKey);
byte[] isNull = {0};
baos.write(KvStateSerializer.serializeValue(expectedKey, userKeySerializer));
baos.write(isNull);
baos.write(KvStateSerializer.serializeValue(expectedValue, userValueSerializer));
byte[] serializedValue = baos.toByteArray();
Map<Long, String> actualValue = KvStateSerializer.deserializeMap(serializedValue, userKeySerializer, userValueSerializer);
assertEquals(1, actualValue.size());
assertEquals(expectedValue, actualValue.get(expectedKey));
}
Example 5
| Source File: HeapKeyedStateBackendSnapshotMigrationTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testMapStateMigrationAfterHashMapSerRemoval() throws Exception {
ClassLoader cl = getClass().getClassLoader();
URL resource = cl.getResource("heap_keyed_statebackend_1_5_map.snapshot");
Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");
final SnapshotResult<KeyedStateHandle> stateHandles;
try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
stateHandles = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
}
final KeyedStateHandle stateHandle = stateHandles.getJobManagerOwnedSnapshot();
try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
final Integer namespace1 = 1;
final Integer namespace2 = 2;
final Integer namespace3 = 3;
final MapStateDescriptor<Long, Long> stateDescr = new MapStateDescriptor<>("my-map-state", Long.class, Long.class);
stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());
InternalMapState<String, Integer, Long, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);
keyedBackend.setCurrentKey("abc");
state.setCurrentNamespace(namespace1);
assertEquals(33L, (long) state.get(33L));
assertEquals(55L, (long) state.get(55L));
assertEquals(2, getStateSize(state));
state.setCurrentNamespace(namespace2);
assertEquals(22L, (long) state.get(22L));
assertEquals(11L, (long) state.get(11L));
assertEquals(2, getStateSize(state));
state.setCurrentNamespace(namespace3);
assertEquals(44L, (long) state.get(44L));
assertEquals(1, getStateSize(state));
keyedBackend.setCurrentKey("def");
state.setCurrentNamespace(namespace1);
assertEquals(11L, (long) state.get(11L));
assertEquals(44L, (long) state.get(44L));
assertEquals(2, getStateSize(state));
state.setCurrentNamespace(namespace3);
assertEquals(22L, (long) state.get(22L));
assertEquals(55L, (long) state.get(55L));
assertEquals(33L, (long) state.get(33L));
assertEquals(3, getStateSize(state));
keyedBackend.setCurrentKey("jkl");
state.setCurrentNamespace(namespace1);
assertEquals(11L, (long) state.get(11L));
assertEquals(22L, (long) state.get(22L));
assertEquals(33L, (long) state.get(33L));
assertEquals(44L, (long) state.get(44L));
assertEquals(55L, (long) state.get(55L));
assertEquals(5, getStateSize(state));
keyedBackend.setCurrentKey("mno");
state.setCurrentNamespace(namespace3);
assertEquals(11L, (long) state.get(11L));
assertEquals(22L, (long) state.get(22L));
assertEquals(33L, (long) state.get(33L));
assertEquals(44L, (long) state.get(44L));
assertEquals(55L, (long) state.get(55L));
assertEquals(5, getStateSize(state));
RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = keyedBackend.snapshot(
1L,
1L,
new MemCheckpointStreamFactory(4 * 1024 * 1024),
CheckpointOptions.forCheckpointWithDefaultLocation());
snapshot.run();
}
}
Example 6
| Source File: KvStateRequestSerializerTest.java From flink with Apache License 2.0 | 4 votes |
|
/**
* Verifies that the serialization of a map using the given map state
* matches the deserialization with {@link KvStateSerializer#deserializeList}.
*
* @param key
* key of the map state
* @param mapState
* map state using the {@link VoidNamespace}, must also be a {@link InternalKvState} instance
*
* @throws Exception
*/
public static void testMapSerialization(
final long key,
final InternalMapState<Long, VoidNamespace, Long, String> mapState) throws Exception {
TypeSerializer<Long> userKeySerializer = LongSerializer.INSTANCE;
TypeSerializer<String> userValueSerializer = StringSerializer.INSTANCE;
mapState.setCurrentNamespace(VoidNamespace.INSTANCE);
// Map
final int numElements = 10;
final Map<Long, String> expectedValues = new HashMap<>();
for (int i = 1; i <= numElements; i++) {
final long value = ThreadLocalRandom.current().nextLong();
expectedValues.put(value, Long.toString(value));
mapState.put(value, Long.toString(value));
}
expectedValues.put(0L, null);
mapState.put(0L, null);
final byte[] serializedKey =
KvStateSerializer.serializeKeyAndNamespace(
key, LongSerializer.INSTANCE,
VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
final byte[] serializedValues = mapState.getSerializedValue(
serializedKey,
mapState.getKeySerializer(),
mapState.getNamespaceSerializer(),
mapState.getValueSerializer());
Map<Long, String> actualValues = KvStateSerializer.deserializeMap(serializedValues, userKeySerializer, userValueSerializer);
assertEquals(expectedValues.size(), actualValues.size());
for (Map.Entry<Long, String> actualEntry : actualValues.entrySet()) {
assertEquals(expectedValues.get(actualEntry.getKey()), actualEntry.getValue());
}
// Single value
ByteArrayOutputStream baos = new ByteArrayOutputStream();
long expectedKey = ThreadLocalRandom.current().nextLong();
String expectedValue = Long.toString(expectedKey);
byte[] isNull = {0};
baos.write(KvStateSerializer.serializeValue(expectedKey, userKeySerializer));
baos.write(isNull);
baos.write(KvStateSerializer.serializeValue(expectedValue, userValueSerializer));
byte[] serializedValue = baos.toByteArray();
Map<Long, String> actualValue = KvStateSerializer.deserializeMap(serializedValue, userKeySerializer, userValueSerializer);
assertEquals(1, actualValue.size());
assertEquals(expectedValue, actualValue.get(expectedKey));
}
