org.apache.flink.runtime.state.StateEntry Java Examples

private void runCleanup() {
	int entryNum = 0;
	Collection<StateEntry<K, N, S>> nextEntries;
	while (
		entryNum < cleanupSize &&
		stateIterator.hasNext() &&
		!(nextEntries = stateIterator.nextEntries()).isEmpty()) {

		for (StateEntry<K, N, S> state : nextEntries) {
			S cleanState = ttlState.getUnexpiredOrNull(state.getState());
			if (cleanState == null) {
				stateIterator.remove(state);
			} else if (cleanState != state.getState()) {
				stateIterator.update(state, cleanState);
			}
		}

		entryNum += nextEntries.size();
	}
}
 

@Nonnull
@Override
public Iterator<StateEntry<K, N, S>> iterator() {
	updateStat();
	final Iterator<Long> nodeIter = new NodeIterator();
	return new Iterator<StateEntry<K, N, S>>() {
		@Override
		public boolean hasNext() {
			return nodeIter.hasNext();
		}

		@Override
		public StateEntry<K, N, S> next() {
			return helpGetStateEntry(nodeIter.next());
		}
	};
}
 

/**
 * Returns the state entry of the node.
 */
private StateEntry<K, N, S> helpGetStateEntry(long node) {
	Node nodeStorage = getNodeSegmentAndOffset(node);
	MemorySegment segment = nodeStorage.nodeSegment;
	int offsetInSegment = nodeStorage.nodeOffset;

	int level = SkipListUtils.getLevel(segment, offsetInSegment);
	int keyDataLen = SkipListUtils.getKeyLen(segment, offsetInSegment);
	int keyDataOffset = offsetInSegment + SkipListUtils.getKeyDataOffset(level);

	K key = skipListKeySerializer.deserializeKey(segment, keyDataOffset, keyDataLen);
	N namespace = skipListKeySerializer.deserializeNamespace(segment, keyDataOffset, keyDataLen);
	long valuePointer = SkipListUtils.getValuePointer(segment, offsetInSegment);
	S state = helpGetState(valuePointer);

	return new StateEntry.SimpleStateEntry<>(key, namespace, state);
}
 

private void runCleanup() {
	int entryNum = 0;
	Collection<StateEntry<K, N, S>> nextEntries;
	while (
		entryNum < cleanupSize &&
		stateIterator.hasNext() &&
		!(nextEntries = stateIterator.nextEntries()).isEmpty()) {

		for (StateEntry<K, N, S> state : nextEntries) {
			S cleanState = ttlState.getUnexpiredOrNull(state.getState());
			if (cleanState == null) {
				stateIterator.remove(state);
			} else if (cleanState != state.getState()) {
				stateIterator.update(state, cleanState);
			}
		}

		entryNum += nextEntries.size();
	}
}
 

@Override
public void writeState(
	TypeSerializer<K> keySerializer,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<S> stateSerializer,
	@Nonnull DataOutputView dov,
	@Nullable StateSnapshotTransformer<S> stateSnapshotTransformer) throws IOException {
	SnapshotIterator<K, N, S> snapshotIterator = stateSnapshotTransformer == null ?
		new NonTransformSnapshotIterator<>(numberOfEntriesInSnapshotData, snapshotData) :
		new TransformedSnapshotIterator<>(numberOfEntriesInSnapshotData, snapshotData, stateSnapshotTransformer);

	int size = snapshotIterator.size();
	dov.writeInt(size);
	while (snapshotIterator.hasNext()) {
		StateEntry<K, N, S> stateEntry = snapshotIterator.next();
		namespaceSerializer.serialize(stateEntry.getNamespace(), dov);
		keySerializer.serialize(stateEntry.getKey(), dov);
		stateSerializer.serialize(stateEntry.getState(), dov);
	}
}
 

@Override
public void writeState(
	TypeSerializer<K> keySerializer,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<S> stateSerializer,
	@Nonnull DataOutputView dov,
	@Nullable StateSnapshotTransformer<S> stateSnapshotTransformer) throws IOException {
	SnapshotIterator<K, N, S> snapshotIterator = stateSnapshotTransformer == null ?
		new NonTransformSnapshotIterator<>(numberOfEntriesInSnapshotData, snapshotData) :
		new TransformedSnapshotIterator<>(numberOfEntriesInSnapshotData, snapshotData, stateSnapshotTransformer);

	int size = snapshotIterator.size();
	dov.writeInt(size);
	while (snapshotIterator.hasNext()) {
		StateEntry<K, N, S> stateEntry = snapshotIterator.next();
		namespaceSerializer.serialize(stateEntry.getNamespace(), dov);
		keySerializer.serialize(stateEntry.getKey(), dov);
		stateSerializer.serialize(stateEntry.getState(), dov);
	}
}
 

@Override
public StateEntry<K, N, S> next() {
	if (!hasNext()) {
		throw new NoSuchElementException();
	}

	if (!keyValueIterator.hasNext()) {
		namespace = namespaceIterator.next();
		keyValueIterator = namespace.getValue().entrySet().iterator();
	}

	Map.Entry<K, S> entry = keyValueIterator.next();

	return new StateEntry.SimpleStateEntry<>(
		entry.getKey(), namespace.getKey(), entry.getValue());
}
 

@Override
public StateEntry<K, N, S> next() {
	if (!hasNext()) {
		throw new NoSuchElementException();
	}

	if (!keyValueIterator.hasNext()) {
		namespace = namespaceIterator.next();
		keyValueIterator = namespace.getValue().entrySet().iterator();
	}

	Map.Entry<K, S> entry = keyValueIterator.next();

	return new StateEntry.SimpleStateEntry<>(
		entry.getKey(), namespace.getKey(), entry.getValue());
}
 

private void runCleanup() {
	int entryNum = 0;
	Collection<StateEntry<K, N, S>> nextEntries;
	while (
		entryNum < cleanupSize &&
		stateIterator.hasNext() &&
		!(nextEntries = stateIterator.nextEntries()).isEmpty()) {

		for (StateEntry<K, N, S> state : nextEntries) {
			S cleanState = ttlState.getUnexpiredOrNull(state.getState());
			if (cleanState == null) {
				stateIterator.remove(state);
			} else if (cleanState != state.getState()) {
				stateIterator.update(state, cleanState);
			}
		}

		entryNum += nextEntries.size();
	}
}
 

@Override
public StateEntry<K, N, S> next() {
	if (modCount != expectedModCount) {
		throw new ConcurrentModificationException();
	}
	if (!hasNext()) {
		throw new NoSuchElementException();
	}
	return advanceIterator();
}
 

@Override
public StateEntry<K, N, S> next() {
	if (modCount != expectedModCount) {
		throw new ConcurrentModificationException();
	}
	if (!hasNext()) {
		throw new NoSuchElementException();
	}
	return advanceIterator();
}
 

/**
 * Test operations specific for StateIncrementalVisitor in {@code testRandomModificationsAndCopyOnWriteIsolation()}.
 *
 * <p>Check next, update and remove during global iteration of StateIncrementalVisitor.
 */
private static void testStateIteratorWithUpdate(
	StateIncrementalVisitor<Integer, Integer, ArrayList<Integer>> updatingIterator,
	CopyOnWriteStateMap<Integer, Integer, ArrayList<Integer>> stateMap,
	HashMap<Tuple2<Integer, Integer>, ArrayList<Integer>> referenceMap,
	boolean update, boolean remove) {

	for (StateEntry<Integer, Integer, ArrayList<Integer>> stateEntry : updatingIterator.nextEntries()) {
		Integer key = stateEntry.getKey();
		Integer namespace = stateEntry.getNamespace();
		Tuple2<Integer, Integer> compositeKey = new Tuple2<>(key, namespace);
		Assert.assertEquals(referenceMap.get(compositeKey), stateEntry.getState());

		if (update) {
			ArrayList<Integer> newState = new ArrayList<>(stateEntry.getState());
			if (!newState.isEmpty()) {
				newState.remove(0);
			}
			updatingIterator.update(stateEntry, newState);
			referenceMap.put(compositeKey, new ArrayList<>(newState));
			Assert.assertEquals(newState, stateMap.get(key, namespace));
		}

		if (remove) {
			updatingIterator.remove(stateEntry);
			referenceMap.remove(compositeKey);
		}
	}
}
 

@Override
public Collection<StateEntry<K, N, S>> nextEntries() {
	if (!hasNext()) {
		return null;
	}

	chainToReturn.clear();
	for (StateTableEntry<K, N, S> nextEntry = chainIterator.next();
		 nextEntry != null;
		 nextEntry = nextEntry.next) {
		chainToReturn.add(nextEntry);
	}
	return chainToReturn;
}
 

private boolean keyIteratorHasNext() {
	while (nextEntry == null && keyValueIterator != null && keyValueIterator.hasNext()) {
		Map.Entry<K, S> next = keyValueIterator.next();
		Map<K, S> ns = namespaceMap.getOrDefault(namespace.getKey(), null);
		S upToDateValue = ns == null ? null : ns.getOrDefault(next.getKey(), null);
		if (upToDateValue != null) {
			nextEntry = new StateEntry.SimpleStateEntry<>(next.getKey(), namespace.getKey(), upToDateValue);
		}
	}
	return nextEntry != null;
}
 

@Override
public int sizeOfNamespace(Object namespace) {
	int count = 0;
	for (StateEntry<K, N, S> entry : this) {
		if (null != entry && namespace.equals(entry.getNamespace())) {
			++count;
		}
	}
	return count;
}
 

@Override
public Collection<StateEntry<K, N, S>> nextEntries() {
	if (!hasNext()) {
		return null;
	}

	chainToReturn.clear();
	for (StateMapEntry<K, N, S> nextEntry = chainIterator.next();
			nextEntry != null;
			nextEntry = nextEntry.next) {
		chainToReturn.add(nextEntry);
	}
	return chainToReturn;
}
 

/**
 * Test operations specific for StateIncrementalVisitor in {@code testRandomModificationsAndCopyOnWriteIsolation()}.
 *
 * <p>Check next, update and remove during global iteration of StateIncrementalVisitor.
 */
private static void testStateIteratorWithUpdate(
	StateIncrementalVisitor<Integer, Integer, ArrayList<Integer>> updatingIterator,
	CopyOnWriteStateTable<Integer, Integer, ArrayList<Integer>> stateTable,
	HashMap<Tuple2<Integer, Integer>, ArrayList<Integer>> referenceMap,
	boolean update, boolean remove) {

	for (StateEntry<Integer, Integer, ArrayList<Integer>> stateEntry : updatingIterator.nextEntries()) {
		Integer key = stateEntry.getKey();
		Integer namespace = stateEntry.getNamespace();
		Tuple2<Integer, Integer> compositeKey = new Tuple2<>(key, namespace);
		Assert.assertEquals(referenceMap.get(compositeKey), stateEntry.getState());

		if (update) {
			ArrayList<Integer> newState = new ArrayList<>(stateEntry.getState());
			if (!newState.isEmpty()) {
				newState.remove(0);
			}
			updatingIterator.update(stateEntry, newState);
			referenceMap.put(compositeKey, new ArrayList<>(newState));
			Assert.assertEquals(newState, stateTable.get(key, namespace));
		}

		if (remove) {
			updatingIterator.remove(stateEntry);
			referenceMap.remove(compositeKey);
		}
	}
}
 

@Override
public int sizeOfNamespace(Object namespace) {
	int count = 0;
	for (StateEntry<K, N, S> entry : this) {
		if (null != entry && namespace.equals(entry.getNamespace())) {
			++count;
		}
	}
	return count;
}
 

@Override
public final boolean equals(Object o) {
	if (!(o instanceof CopyOnWriteStateTable.StateTableEntry)) {
		return false;
	}

	StateEntry<?, ?, ?> e = (StateEntry<?, ?, ?>) o;
	return e.getKey().equals(key)
			&& e.getNamespace().equals(namespace)
			&& Objects.equals(e.getState(), state);
}
 

@Override
public final boolean equals(Object o) {
	if (!(o instanceof CopyOnWriteStateMap.StateMapEntry)) {
		return false;
	}

	StateEntry<?, ?, ?> e = (StateEntry<?, ?, ?>) o;
	return e.getKey().equals(key)
		&& e.getNamespace().equals(namespace)
		&& Objects.equals(e.getState(), state);
}
 

@Override
public Iterator<StateEntry<K, N, S>> iterator() {
	return Arrays.stream(keyGroupedStateMaps)
		.filter(Objects::nonNull)
		.flatMap(stateMap -> StreamSupport.stream(Spliterators.spliteratorUnknownSize(stateMap.iterator(), 0), false))
		.iterator();
}
 

@Override
public Iterator<StateEntry<K, N, S>> iterator() {
	return Arrays.stream(keyGroupedStateMaps)
		.filter(Objects::nonNull)
		.flatMap(stateMap -> StreamSupport.stream(Spliterators.spliteratorUnknownSize(stateMap.iterator(), 0), false))
		.iterator();
}
 

@Override
public Collection<StateEntry<K, N, S>> nextEntries() {
	if (!hasNext()) {
		return null;
	}

	return stateIncrementalVisitor.nextEntries();
}
 

@Override
public Collection<StateEntry<K, N, S>> nextEntries() {
	if (!hasNext()) {
		return null;
	}

	chainToReturn.clear();
	for (StateMapEntry<K, N, S> nextEntry = chainIterator.next();
			nextEntry != null;
			nextEntry = nextEntry.next) {
		chainToReturn.add(nextEntry);
	}
	return chainToReturn;
}
 

@Override
public StateEntry<K, N, S> next() {
	if (modCount != expectedModCount) {
		throw new ConcurrentModificationException();
	}
	if (!hasNext()) {
		throw new NoSuchElementException();
	}
	return advanceIterator();
}
 

@Override
public int sizeOfNamespace(Object namespace) {
	int count = 0;
	for (StateEntry<K, N, S> entry : this) {
		if (null != entry && namespace.equals(entry.getNamespace())) {
			++count;
		}
	}
	return count;
}
 

private boolean keyIteratorHasNext() {
	while (nextEntry == null && keyValueIterator != null && keyValueIterator.hasNext()) {
		Map.Entry<K, S> next = keyValueIterator.next();
		Map<K, S> ns = namespaceMap.getOrDefault(namespace.getKey(), null);
		S upToDateValue = ns == null ? null : ns.getOrDefault(next.getKey(), null);
		if (upToDateValue != null) {
			nextEntry = new StateEntry.SimpleStateEntry<>(next.getKey(), namespace.getKey(), upToDateValue);
		}
	}
	return nextEntry != null;
}
 

@Override
public final boolean equals(Object o) {
	if (!(o instanceof CopyOnWriteStateMap.StateMapEntry)) {
		return false;
	}

	StateEntry<?, ?, ?> e = (StateEntry<?, ?, ?>) o;
	return e.getKey().equals(key)
		&& e.getNamespace().equals(namespace)
		&& Objects.equals(e.getState(), state);
}
 

/**
 * Test operations specific for StateIncrementalVisitor in {@code testRandomModificationsAndCopyOnWriteIsolation()}.
 *
 * <p>Check next, update and remove during global iteration of StateIncrementalVisitor.
 */
private static void testStateIteratorWithUpdate(
	StateIncrementalVisitor<Integer, Integer, ArrayList<Integer>> updatingIterator,
	CopyOnWriteStateMap<Integer, Integer, ArrayList<Integer>> stateMap,
	HashMap<Tuple2<Integer, Integer>, ArrayList<Integer>> referenceMap,
	boolean update, boolean remove) {

	for (StateEntry<Integer, Integer, ArrayList<Integer>> stateEntry : updatingIterator.nextEntries()) {
		Integer key = stateEntry.getKey();
		Integer namespace = stateEntry.getNamespace();
		Tuple2<Integer, Integer> compositeKey = new Tuple2<>(key, namespace);
		Assert.assertEquals(referenceMap.get(compositeKey), stateEntry.getState());

		if (update) {
			ArrayList<Integer> newState = new ArrayList<>(stateEntry.getState());
			if (!newState.isEmpty()) {
				newState.remove(0);
			}
			updatingIterator.update(stateEntry, newState);
			referenceMap.put(compositeKey, new ArrayList<>(newState));
			Assert.assertEquals(newState, stateMap.get(key, namespace));
		}

		if (remove) {
			updatingIterator.remove(stateEntry);
			referenceMap.remove(compositeKey);
		}
	}
}
 

@Override
public Collection<StateEntry<K, N, S>> nextEntries() {
	if (nextKeySegment == null) {
		return Collections.emptyList();
	}

	long node = findNextNode(nextKeySegment, nextKeyOffset);
	if (node == NIL_NODE) {
		nextKeySegment = null;
		return Collections.emptyList();
	}

	entryToReturn.clear();
	entryToReturn.add(helpGetStateEntry(node));
	int n = 1;
	while (n < recommendedMaxNumberOfReturnedRecords) {
		node = getNextNode(node);
		if (node == NIL_NODE) {
			break;
		}
		entryToReturn.add(helpGetStateEntry(node));
		n++;
	}

	updateNextKeySegment(node);

	return entryToReturn;
}