Java Code Examples for org.apache.flink.api.common.state.MapState#iterator()

The following examples show how to use org.apache.flink.api.common.state.MapState#iterator() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: StateBackendTestBase.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Verify iterator of {@link MapState} supporting arbitrary access, see [FLINK-10267] to know more details.
 */
@Test
public void testMapStateIteratorArbitraryAccess() throws Exception {
	MapStateDescriptor<Integer, Long> kvId = new MapStateDescriptor<>("id", Integer.class, Long.class);

	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		MapState<Integer, Long> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
		backend.setCurrentKey(1);
		int stateSize = 4096;
		for (int i = 0; i < stateSize; i++) {
			state.put(i, i * 2L);
		}
		Iterator<Map.Entry<Integer, Long>> iterator = state.iterator();
		int iteratorCount = 0;
		while (iterator.hasNext()) {
			Map.Entry<Integer, Long> entry = iterator.next();
			assertEquals(iteratorCount, (int) entry.getKey());
			switch (ThreadLocalRandom.current().nextInt() % 3) {
				case 0: // remove twice
					iterator.remove();
					try {
						iterator.remove();
						fail();
					} catch (IllegalStateException e) {
						// ignore expected exception
					}
					break;
				case 1: // hasNext -> remove
					iterator.hasNext();
					iterator.remove();
					break;
				case 2: // nothing to do
					break;
			}
			iteratorCount++;
		}
		assertEquals(stateSize, iteratorCount);
	} finally {
		backend.dispose();
	}
}

Example 2

Source File: StateBackendTestBase.java From flink with Apache License 2.0

4 votes

/**
 * Verify iterator of {@link MapState} supporting arbitrary access, see [FLINK-10267] to know more details.
 */
@Test
public void testMapStateIteratorArbitraryAccess() throws Exception {
	MapStateDescriptor<Integer, Long> kvId = new MapStateDescriptor<>("id", Integer.class, Long.class);

	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		MapState<Integer, Long> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
		backend.setCurrentKey(1);
		int stateSize = 4096;
		for (int i = 0; i < stateSize; i++) {
			state.put(i, i * 2L);
		}
		Iterator<Map.Entry<Integer, Long>> iterator = state.iterator();
		int iteratorCount = 0;
		while (iterator.hasNext()) {
			Map.Entry<Integer, Long> entry = iterator.next();
			assertEquals(iteratorCount, (int) entry.getKey());
			switch (ThreadLocalRandom.current().nextInt() % 3) {
				case 0: // remove twice
					iterator.remove();
					try {
						iterator.remove();
						fail();
					} catch (IllegalStateException e) {
						// ignore expected exception
					}
					break;
				case 1: // hasNext -> remove
					iterator.hasNext();
					iterator.remove();
					break;
				case 2: // nothing to do
					break;
			}
			iteratorCount++;
		}
		assertEquals(stateSize, iteratorCount);
	} finally {
		backend.dispose();
	}
}

Example 3

Source File: StateBackendMigrationTestBase.java From flink with Apache License 2.0

4 votes

private void testKeyedMapStateUpgrade(
	MapStateDescriptor<Integer, TestType> initialAccessDescriptor,
	MapStateDescriptor<Integer, TestType> newAccessDescriptorAfterRestore) throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();

	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		MapState<Integer, TestType> mapState = backend.getPartitionedState(
			VoidNamespace.INSTANCE,
			CustomVoidNamespaceSerializer.INSTANCE,
			initialAccessDescriptor);

		backend.setCurrentKey(1);
		mapState.put(1, new TestType("key-1", 1));
		mapState.put(2, new TestType("key-1", 2));
		mapState.put(3, new TestType("key-1", 3));

		backend.setCurrentKey(2);
		mapState.put(1, new TestType("key-2", 1));

		backend.setCurrentKey(3);
		mapState.put(1, new TestType("key-3", 1));
		mapState.put(2, new TestType("key-3", 2));

		KeyedStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);
		backend.dispose();

		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot);

		mapState = backend.getPartitionedState(
			VoidNamespace.INSTANCE,
			CustomVoidNamespaceSerializer.INSTANCE,
			newAccessDescriptorAfterRestore);

		// make sure that reading and writing each key state works with the new serializer
		backend.setCurrentKey(1);
		Iterator<Map.Entry<Integer, TestType>> iterable1 = mapState.iterator();
		Map.Entry<Integer, TestType> actual = iterable1.next();
		assertEquals((Integer) 1, actual.getKey());
		assertEquals(new TestType("key-1", 1), actual.getValue());

		actual = iterable1.next();
		assertEquals((Integer) 2, actual.getKey());
		assertEquals(new TestType("key-1", 2), actual.getValue());

		actual = iterable1.next();
		assertEquals((Integer) 3, actual.getKey());
		assertEquals(new TestType("key-1", 3), actual.getValue());

		Assert.assertFalse(iterable1.hasNext());

		mapState.put(123, new TestType("new-key-1", 123));

		backend.setCurrentKey(2);
		Iterator<Map.Entry<Integer, TestType>> iterable2 = mapState.iterator();

		actual = iterable2.next();
		assertEquals((Integer) 1, actual.getKey());
		assertEquals(new TestType("key-2", 1), actual.getValue());
		Assert.assertFalse(iterable2.hasNext());

		mapState.put(456, new TestType("new-key-2", 456));

		backend.setCurrentKey(3);
		Iterator<Map.Entry<Integer, TestType>> iterable3 = mapState.iterator();

		actual = iterable3.next();
		assertEquals((Integer) 1, actual.getKey());
		assertEquals(new TestType("key-3", 1), actual.getValue());

		actual = iterable3.next();
		assertEquals((Integer) 2, actual.getKey());
		assertEquals(new TestType("key-3", 2), actual.getValue());

		Assert.assertFalse(iterable3.hasNext());
		mapState.put(777, new TestType("new-key-3", 777));

		// do another snapshot to verify the snapshot logic after migration
		snapshot = runSnapshot(
			backend.snapshot(2L, 3L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);
		snapshot.discardState();

	} finally {
		backend.dispose();
	}
}

Example 4

Source File: StateBackendTestBase.java From flink with Apache License 2.0

4 votes

/**
 * Verify iterator of {@link MapState} supporting arbitrary access, see [FLINK-10267] to know more details.
 */
@Test
public void testMapStateIteratorArbitraryAccess() throws Exception {
	MapStateDescriptor<Integer, Long> kvId = new MapStateDescriptor<>("id", Integer.class, Long.class);

	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		MapState<Integer, Long> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
		backend.setCurrentKey(1);
		int stateSize = 4096;
		for (int i = 0; i < stateSize; i++) {
			state.put(i, i * 2L);
		}
		Iterator<Map.Entry<Integer, Long>> iterator = state.iterator();
		int iteratorCount = 0;
		while (iterator.hasNext()) {
			Map.Entry<Integer, Long> entry = iterator.next();
			assertEquals(iteratorCount, (int) entry.getKey());
			switch (ThreadLocalRandom.current().nextInt() % 3) {
				case 0: // remove twice
					iterator.remove();
					try {
						iterator.remove();
						fail();
					} catch (IllegalStateException e) {
						// ignore expected exception
					}
					break;
				case 1: // hasNext -> remove
					iterator.hasNext();
					iterator.remove();
					break;
				case 2: // nothing to do
					break;
			}
			iteratorCount++;
		}
		assertEquals(stateSize, iteratorCount);
	} finally {
		backend.dispose();
	}
}

Example 5

Source File: StateBackendMigrationTestBase.java From flink with Apache License 2.0

4 votes

private void testKeyedMapStateUpgrade(
	MapStateDescriptor<Integer, TestType> initialAccessDescriptor,
	MapStateDescriptor<Integer, TestType> newAccessDescriptorAfterRestore) throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();

	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		MapState<Integer, TestType> mapState = backend.getPartitionedState(
			VoidNamespace.INSTANCE,
			CustomVoidNamespaceSerializer.INSTANCE,
			initialAccessDescriptor);

		backend.setCurrentKey(1);
		mapState.put(1, new TestType("key-1", 1));
		mapState.put(2, new TestType("key-1", 2));
		mapState.put(3, new TestType("key-1", 3));

		backend.setCurrentKey(2);
		mapState.put(1, new TestType("key-2", 1));

		backend.setCurrentKey(3);
		mapState.put(1, new TestType("key-3", 1));
		mapState.put(2, new TestType("key-3", 2));

		KeyedStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);
		backend.dispose();

		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot);

		mapState = backend.getPartitionedState(
			VoidNamespace.INSTANCE,
			CustomVoidNamespaceSerializer.INSTANCE,
			newAccessDescriptorAfterRestore);

		// make sure that reading and writing each key state works with the new serializer
		backend.setCurrentKey(1);
		Iterator<Map.Entry<Integer, TestType>> iterable1 = mapState.iterator();
		Map.Entry<Integer, TestType> actual = iterable1.next();
		assertEquals((Integer) 1, actual.getKey());
		assertEquals(new TestType("key-1", 1), actual.getValue());

		actual = iterable1.next();
		assertEquals((Integer) 2, actual.getKey());
		assertEquals(new TestType("key-1", 2), actual.getValue());

		actual = iterable1.next();
		assertEquals((Integer) 3, actual.getKey());
		assertEquals(new TestType("key-1", 3), actual.getValue());

		Assert.assertFalse(iterable1.hasNext());

		mapState.put(123, new TestType("new-key-1", 123));

		backend.setCurrentKey(2);
		Iterator<Map.Entry<Integer, TestType>> iterable2 = mapState.iterator();

		actual = iterable2.next();
		assertEquals((Integer) 1, actual.getKey());
		assertEquals(new TestType("key-2", 1), actual.getValue());
		Assert.assertFalse(iterable2.hasNext());

		mapState.put(456, new TestType("new-key-2", 456));

		backend.setCurrentKey(3);
		Iterator<Map.Entry<Integer, TestType>> iterable3 = mapState.iterator();

		actual = iterable3.next();
		assertEquals((Integer) 1, actual.getKey());
		assertEquals(new TestType("key-3", 1), actual.getValue());

		actual = iterable3.next();
		assertEquals((Integer) 2, actual.getKey());
		assertEquals(new TestType("key-3", 2), actual.getValue());

		Assert.assertFalse(iterable3.hasNext());
		mapState.put(777, new TestType("new-key-3", 777));

		// do another snapshot to verify the snapshot logic after migration
		snapshot = runSnapshot(
			backend.snapshot(2L, 3L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);
		snapshot.discardState();

	} finally {
		backend.dispose();
	}
}