Java Code Examples for org.apache.flink.runtime.state.KeyedStateHandle#getClass()
The following examples show how to use
org.apache.flink.runtime.state.KeyedStateHandle#getClass() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: RocksDBIncrementalRestoreOperation.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Recovery from a single remote incremental state without rescaling. */ private void restoreWithoutRescaling(KeyedStateHandle keyedStateHandle) throws Exception { if (keyedStateHandle instanceof IncrementalRemoteKeyedStateHandle) { IncrementalRemoteKeyedStateHandle incrementalRemoteKeyedStateHandle = (IncrementalRemoteKeyedStateHandle) keyedStateHandle; restorePreviousIncrementalFilesStatus(incrementalRemoteKeyedStateHandle); restoreFromRemoteState(incrementalRemoteKeyedStateHandle); } else if (keyedStateHandle instanceof IncrementalLocalKeyedStateHandle) { IncrementalLocalKeyedStateHandle incrementalLocalKeyedStateHandle = (IncrementalLocalKeyedStateHandle) keyedStateHandle; restorePreviousIncrementalFilesStatus(incrementalLocalKeyedStateHandle); restoreFromLocalState(incrementalLocalKeyedStateHandle); } else { throw new BackendBuildingException("Unexpected state handle type, " + "expected " + IncrementalRemoteKeyedStateHandle.class + " or " + IncrementalLocalKeyedStateHandle.class + ", but found " + keyedStateHandle.getClass()); } }
Example 2
Source File: StreamTaskStateInitializerImpl.java From flink with Apache License 2.0 | 6 votes |
private static Collection<KeyGroupsStateHandle> transform(Collection<KeyedStateHandle> keyedStateHandles) { if (keyedStateHandles == null) { return null; } List<KeyGroupsStateHandle> keyGroupsStateHandles = new ArrayList<>(keyedStateHandles.size()); for (KeyedStateHandle keyedStateHandle : keyedStateHandles) { if (keyedStateHandle instanceof KeyGroupsStateHandle) { keyGroupsStateHandles.add((KeyGroupsStateHandle) keyedStateHandle); } else if (keyedStateHandle != null) { throw new IllegalStateException("Unexpected state handle type, " + "expected: " + KeyGroupsStateHandle.class + ", but found: " + keyedStateHandle.getClass() + "."); } } return keyGroupsStateHandles; }
Example 3
Source File: RocksDBIncrementalRestoreOperation.java From flink with Apache License 2.0 | 6 votes |
/** * Recovery from a single remote incremental state without rescaling. */ private void restoreWithoutRescaling(KeyedStateHandle keyedStateHandle) throws Exception { if (keyedStateHandle instanceof IncrementalRemoteKeyedStateHandle) { IncrementalRemoteKeyedStateHandle incrementalRemoteKeyedStateHandle = (IncrementalRemoteKeyedStateHandle) keyedStateHandle; restorePreviousIncrementalFilesStatus(incrementalRemoteKeyedStateHandle); restoreFromRemoteState(incrementalRemoteKeyedStateHandle); } else if (keyedStateHandle instanceof IncrementalLocalKeyedStateHandle) { IncrementalLocalKeyedStateHandle incrementalLocalKeyedStateHandle = (IncrementalLocalKeyedStateHandle) keyedStateHandle; restorePreviousIncrementalFilesStatus(incrementalLocalKeyedStateHandle); restoreFromLocalState(incrementalLocalKeyedStateHandle); } else { throw new BackendBuildingException("Unexpected state handle type, " + "expected " + IncrementalRemoteKeyedStateHandle.class + " or " + IncrementalLocalKeyedStateHandle.class + ", but found " + keyedStateHandle.getClass()); } }
Example 4
Source File: RocksDBFullRestoreOperation.java From flink with Apache License 2.0 | 6 votes |
/** * Restores all key-groups data that is referenced by the passed state handles. * */ @Override public RocksDBRestoreResult restore() throws IOException, StateMigrationException, RocksDBException { openDB(); for (KeyedStateHandle keyedStateHandle : restoreStateHandles) { if (keyedStateHandle != null) { if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) { throw new IllegalStateException("Unexpected state handle type, " + "expected: " + KeyGroupsStateHandle.class + ", but found: " + keyedStateHandle.getClass()); } this.currentKeyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle; restoreKeyGroupsInStateHandle(); } } return new RocksDBRestoreResult(this.db, defaultColumnFamilyHandle, nativeMetricMonitor, -1, null, null); }
Example 5
Source File: StreamTaskStateInitializerImpl.java From flink with Apache License 2.0 | 6 votes |
private static Collection<KeyGroupsStateHandle> transform(Collection<KeyedStateHandle> keyedStateHandles) { if (keyedStateHandles == null) { return null; } List<KeyGroupsStateHandle> keyGroupsStateHandles = new ArrayList<>(keyedStateHandles.size()); for (KeyedStateHandle keyedStateHandle : keyedStateHandles) { if (keyedStateHandle instanceof KeyGroupsStateHandle) { keyGroupsStateHandles.add((KeyGroupsStateHandle) keyedStateHandle); } else if (keyedStateHandle != null) { throw new IllegalStateException("Unexpected state handle type, " + "expected: " + KeyGroupsStateHandle.class + ", but found: " + keyedStateHandle.getClass() + "."); } } return keyGroupsStateHandles; }
Example 6
Source File: SavepointV1Serializer.java From flink with Apache License 2.0 | 6 votes |
@VisibleForTesting public static void serializeKeyedStateHandle( KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException { if (stateHandle == null) { dos.writeByte(NULL_HANDLE); } else if (stateHandle instanceof KeyGroupsStateHandle) { KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle; dos.writeByte(KEY_GROUPS_HANDLE); dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup()); dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups()); for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) { dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup)); } serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos); } else { throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass()); } }
Example 7
Source File: RocksDBIncrementalRestoreOperation.java From flink with Apache License 2.0 | 6 votes |
/** * Recovery from a single remote incremental state without rescaling. */ private void restoreWithoutRescaling(KeyedStateHandle keyedStateHandle) throws Exception { if (keyedStateHandle instanceof IncrementalRemoteKeyedStateHandle) { IncrementalRemoteKeyedStateHandle incrementalRemoteKeyedStateHandle = (IncrementalRemoteKeyedStateHandle) keyedStateHandle; restorePreviousIncrementalFilesStatus(incrementalRemoteKeyedStateHandle); restoreFromRemoteState(incrementalRemoteKeyedStateHandle); } else if (keyedStateHandle instanceof IncrementalLocalKeyedStateHandle) { IncrementalLocalKeyedStateHandle incrementalLocalKeyedStateHandle = (IncrementalLocalKeyedStateHandle) keyedStateHandle; restorePreviousIncrementalFilesStatus(incrementalLocalKeyedStateHandle); restoreFromLocalState(incrementalLocalKeyedStateHandle); } else { throw new BackendBuildingException("Unexpected state handle type, " + "expected " + IncrementalRemoteKeyedStateHandle.class + " or " + IncrementalLocalKeyedStateHandle.class + ", but found " + keyedStateHandle.getClass()); } }
Example 8
Source File: RocksDBFullRestoreOperation.java From flink with Apache License 2.0 | 6 votes |
/** * Restores all key-groups data that is referenced by the passed state handles. * */ @Override public RocksDBRestoreResult restore() throws IOException, StateMigrationException, RocksDBException { openDB(); for (KeyedStateHandle keyedStateHandle : restoreStateHandles) { if (keyedStateHandle != null) { if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) { throw new IllegalStateException("Unexpected state handle type, " + "expected: " + KeyGroupsStateHandle.class + ", but found: " + keyedStateHandle.getClass()); } this.currentKeyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle; restoreKeyGroupsInStateHandle(); } } return new RocksDBRestoreResult(this.db, defaultColumnFamilyHandle, nativeMetricMonitor, -1, null, null); }
Example 9
Source File: StreamTaskStateInitializerImpl.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private static Collection<KeyGroupsStateHandle> transform(Collection<KeyedStateHandle> keyedStateHandles) { if (keyedStateHandles == null) { return null; } List<KeyGroupsStateHandle> keyGroupsStateHandles = new ArrayList<>(keyedStateHandles.size()); for (KeyedStateHandle keyedStateHandle : keyedStateHandles) { if (keyedStateHandle instanceof KeyGroupsStateHandle) { keyGroupsStateHandles.add((KeyGroupsStateHandle) keyedStateHandle); } else if (keyedStateHandle != null) { throw new IllegalStateException("Unexpected state handle type, " + "expected: " + KeyGroupsStateHandle.class + ", but found: " + keyedStateHandle.getClass() + "."); } } return keyGroupsStateHandles; }
Example 10
Source File: SavepointV1Serializer.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@VisibleForTesting public static void serializeKeyedStateHandle( KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException { if (stateHandle == null) { dos.writeByte(NULL_HANDLE); } else if (stateHandle instanceof KeyGroupsStateHandle) { KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle; dos.writeByte(KEY_GROUPS_HANDLE); dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup()); dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups()); for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) { dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup)); } serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos); } else { throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass()); } }
Example 11
Source File: RocksDBFullRestoreOperation.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Restores all key-groups data that is referenced by the passed state handles. * */ @Override public RocksDBRestoreResult restore() throws IOException, StateMigrationException, RocksDBException { openDB(); for (KeyedStateHandle keyedStateHandle : restoreStateHandles) { if (keyedStateHandle != null) { if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) { throw new IllegalStateException("Unexpected state handle type, " + "expected: " + KeyGroupsStateHandle.class + ", but found: " + keyedStateHandle.getClass()); } this.currentKeyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle; restoreKeyGroupsInStateHandle(); } } return new RocksDBRestoreResult(this.db, defaultColumnFamilyHandle, nativeMetricMonitor, -1, null, null); }
Example 12
Source File: SavepointV2Serializer.java From flink with Apache License 2.0 | 5 votes |
@VisibleForTesting public static void serializeKeyedStateHandle( KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException { if (stateHandle == null) { dos.writeByte(NULL_HANDLE); } else if (stateHandle instanceof KeyGroupsStateHandle) { KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle; dos.writeByte(KEY_GROUPS_HANDLE); dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup()); dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups()); for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) { dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup)); } serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos); } else if (stateHandle instanceof IncrementalRemoteKeyedStateHandle) { IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle = (IncrementalRemoteKeyedStateHandle) stateHandle; dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE); dos.writeLong(incrementalKeyedStateHandle.getCheckpointId()); dos.writeUTF(String.valueOf(incrementalKeyedStateHandle.getBackendIdentifier())); dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup()); dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups()); serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos); serializeStreamStateHandleMap(incrementalKeyedStateHandle.getSharedState(), dos); serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos); } else { throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass()); } }
Example 13
Source File: SavepointV2Serializer.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@VisibleForTesting public static void serializeKeyedStateHandle( KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException { if (stateHandle == null) { dos.writeByte(NULL_HANDLE); } else if (stateHandle instanceof KeyGroupsStateHandle) { KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle; dos.writeByte(KEY_GROUPS_HANDLE); dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup()); dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups()); for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) { dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup)); } serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos); } else if (stateHandle instanceof IncrementalRemoteKeyedStateHandle) { IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle = (IncrementalRemoteKeyedStateHandle) stateHandle; dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE); dos.writeLong(incrementalKeyedStateHandle.getCheckpointId()); dos.writeUTF(String.valueOf(incrementalKeyedStateHandle.getBackendIdentifier())); dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup()); dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups()); serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos); serializeStreamStateHandleMap(incrementalKeyedStateHandle.getSharedState(), dos); serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos); } else { throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass()); } }
Example 14
Source File: MetadataV2V3SerializerBase.java From flink with Apache License 2.0 | 5 votes |
void serializeKeyedStateHandle(KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException { if (stateHandle == null) { dos.writeByte(NULL_HANDLE); } else if (stateHandle instanceof KeyGroupsStateHandle) { KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle; dos.writeByte(KEY_GROUPS_HANDLE); dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup()); dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups()); for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) { dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup)); } serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos); } else if (stateHandle instanceof IncrementalRemoteKeyedStateHandle) { IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle = (IncrementalRemoteKeyedStateHandle) stateHandle; dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE); dos.writeLong(incrementalKeyedStateHandle.getCheckpointId()); dos.writeUTF(String.valueOf(incrementalKeyedStateHandle.getBackendIdentifier())); dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup()); dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups()); serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos); serializeStreamStateHandleMap(incrementalKeyedStateHandle.getSharedState(), dos); serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos); } else { throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass()); } }
Example 15
Source File: RocksDBIncrementalRestoreOperation.java From flink with Apache License 2.0 | 4 votes |
/** * Recovery from multi incremental states with rescaling. For rescaling, this method creates a temporary * RocksDB instance for a key-groups shard. All contents from the temporary instance are copied into the * real restore instance and then the temporary instance is discarded. */ private void restoreWithRescaling(Collection<KeyedStateHandle> restoreStateHandles) throws Exception { // Prepare for restore with rescaling KeyedStateHandle initialHandle = RocksDBIncrementalCheckpointUtils.chooseTheBestStateHandleForInitial( restoreStateHandles, keyGroupRange); // Init base DB instance if (initialHandle != null) { restoreStateHandles.remove(initialHandle); initDBWithRescaling(initialHandle); } else { openDB(); } // Transfer remaining key-groups from temporary instance into base DB byte[] startKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes]; RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getStartKeyGroup(), startKeyGroupPrefixBytes); byte[] stopKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes]; RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getEndKeyGroup() + 1, stopKeyGroupPrefixBytes); for (KeyedStateHandle rawStateHandle : restoreStateHandles) { if (!(rawStateHandle instanceof IncrementalRemoteKeyedStateHandle)) { throw new IllegalStateException("Unexpected state handle type, " + "expected " + IncrementalRemoteKeyedStateHandle.class + ", but found " + rawStateHandle.getClass()); } Path temporaryRestoreInstancePath = new Path(instanceBasePath.getAbsolutePath() + UUID.randomUUID().toString()); try (RestoredDBInstance tmpRestoreDBInfo = restoreDBInstanceFromStateHandle( (IncrementalRemoteKeyedStateHandle) rawStateHandle, temporaryRestoreInstancePath); RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(this.db)) { List<ColumnFamilyDescriptor> tmpColumnFamilyDescriptors = tmpRestoreDBInfo.columnFamilyDescriptors; List<ColumnFamilyHandle> tmpColumnFamilyHandles = tmpRestoreDBInfo.columnFamilyHandles; // iterating only the requested descriptors automatically skips the default column family handle for (int i = 0; i < tmpColumnFamilyDescriptors.size(); ++i) { ColumnFamilyHandle tmpColumnFamilyHandle = tmpColumnFamilyHandles.get(i); ColumnFamilyHandle targetColumnFamilyHandle = getOrRegisterStateColumnFamilyHandle( null, tmpRestoreDBInfo.stateMetaInfoSnapshots.get(i)) .columnFamilyHandle; try (RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(tmpRestoreDBInfo.db, tmpColumnFamilyHandle)) { iterator.seek(startKeyGroupPrefixBytes); while (iterator.isValid()) { if (RocksDBIncrementalCheckpointUtils.beforeThePrefixBytes(iterator.key(), stopKeyGroupPrefixBytes)) { writeBatchWrapper.put(targetColumnFamilyHandle, iterator.key(), iterator.value()); } else { // Since the iterator will visit the record according to the sorted order, // we can just break here. break; } iterator.next(); } } // releases native iterator resources } } finally { cleanUpPathQuietly(temporaryRestoreInstancePath); } } }
Example 16
Source File: HeapRestoreOperation.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Override public Void restore() throws Exception { final Map<Integer, StateMetaInfoSnapshot> kvStatesById = new HashMap<>(); registeredKVStates.clear(); registeredPQStates.clear(); boolean keySerializerRestored = false; for (KeyedStateHandle keyedStateHandle : restoreStateHandles) { if (keyedStateHandle == null) { continue; } if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) { throw new IllegalStateException("Unexpected state handle type, " + "expected: " + KeyGroupsStateHandle.class + ", but found: " + keyedStateHandle.getClass()); } KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle; FSDataInputStream fsDataInputStream = keyGroupsStateHandle.openInputStream(); cancelStreamRegistry.registerCloseable(fsDataInputStream); try { DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(fsDataInputStream); KeyedBackendSerializationProxy<K> serializationProxy = new KeyedBackendSerializationProxy<>(userCodeClassLoader); serializationProxy.read(inView); if (!keySerializerRestored) { // check for key serializer compatibility; this also reconfigures the // key serializer to be compatible, if it is required and is possible TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat = keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(serializationProxy.getKeySerializerSnapshot()); if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) { throw new StateMigrationException("The new key serializer must be compatible."); } keySerializerRestored = true; } List<StateMetaInfoSnapshot> restoredMetaInfos = serializationProxy.getStateMetaInfoSnapshots(); createOrCheckStateForMetaInfo(restoredMetaInfos, kvStatesById); readStateHandleStateData( fsDataInputStream, inView, keyGroupsStateHandle.getGroupRangeOffsets(), kvStatesById, restoredMetaInfos.size(), serializationProxy.getReadVersion(), serializationProxy.isUsingKeyGroupCompression()); } finally { if (cancelStreamRegistry.unregisterCloseable(fsDataInputStream)) { IOUtils.closeQuietly(fsDataInputStream); } } } return null; }
Example 17
Source File: HeapRestoreOperation.java From flink with Apache License 2.0 | 4 votes |
@Override public Void restore() throws Exception { final Map<Integer, StateMetaInfoSnapshot> kvStatesById = new HashMap<>(); registeredKVStates.clear(); registeredPQStates.clear(); boolean keySerializerRestored = false; for (KeyedStateHandle keyedStateHandle : restoreStateHandles) { if (keyedStateHandle == null) { continue; } if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) { throw new IllegalStateException("Unexpected state handle type, " + "expected: " + KeyGroupsStateHandle.class + ", but found: " + keyedStateHandle.getClass()); } KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle; FSDataInputStream fsDataInputStream = keyGroupsStateHandle.openInputStream(); cancelStreamRegistry.registerCloseable(fsDataInputStream); try { DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(fsDataInputStream); KeyedBackendSerializationProxy<K> serializationProxy = new KeyedBackendSerializationProxy<>(userCodeClassLoader); serializationProxy.read(inView); if (!keySerializerRestored) { // check for key serializer compatibility; this also reconfigures the // key serializer to be compatible, if it is required and is possible TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat = keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(serializationProxy.getKeySerializerSnapshot()); if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) { throw new StateMigrationException("The new key serializer must be compatible."); } keySerializerRestored = true; } List<StateMetaInfoSnapshot> restoredMetaInfos = serializationProxy.getStateMetaInfoSnapshots(); createOrCheckStateForMetaInfo(restoredMetaInfos, kvStatesById); readStateHandleStateData( fsDataInputStream, inView, keyGroupsStateHandle.getGroupRangeOffsets(), kvStatesById, restoredMetaInfos.size(), serializationProxy.getReadVersion(), serializationProxy.isUsingKeyGroupCompression()); } finally { if (cancelStreamRegistry.unregisterCloseable(fsDataInputStream)) { IOUtils.closeQuietly(fsDataInputStream); } } } return null; }
Example 18
Source File: RocksDBIncrementalRestoreOperation.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
/** * Recovery from multi incremental states with rescaling. For rescaling, this method creates a temporary * RocksDB instance for a key-groups shard. All contents from the temporary instance are copied into the * real restore instance and then the temporary instance is discarded. */ private void restoreWithRescaling(Collection<KeyedStateHandle> restoreStateHandles) throws Exception { // Prepare for restore with rescaling KeyedStateHandle initialHandle = RocksDBIncrementalCheckpointUtils.chooseTheBestStateHandleForInitial( restoreStateHandles, keyGroupRange); // Init base DB instance if (initialHandle != null) { restoreStateHandles.remove(initialHandle); initDBWithRescaling(initialHandle); } else { openDB(); } // Transfer remaining key-groups from temporary instance into base DB byte[] startKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes]; RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getStartKeyGroup(), startKeyGroupPrefixBytes); byte[] stopKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes]; RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getEndKeyGroup() + 1, stopKeyGroupPrefixBytes); for (KeyedStateHandle rawStateHandle : restoreStateHandles) { if (!(rawStateHandle instanceof IncrementalRemoteKeyedStateHandle)) { throw new IllegalStateException("Unexpected state handle type, " + "expected " + IncrementalRemoteKeyedStateHandle.class + ", but found " + rawStateHandle.getClass()); } Path temporaryRestoreInstancePath = new Path(instanceBasePath.getAbsolutePath() + UUID.randomUUID().toString()); try (RestoredDBInstance tmpRestoreDBInfo = restoreDBInstanceFromStateHandle( (IncrementalRemoteKeyedStateHandle) rawStateHandle, temporaryRestoreInstancePath); RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(this.db)) { List<ColumnFamilyDescriptor> tmpColumnFamilyDescriptors = tmpRestoreDBInfo.columnFamilyDescriptors; List<ColumnFamilyHandle> tmpColumnFamilyHandles = tmpRestoreDBInfo.columnFamilyHandles; // iterating only the requested descriptors automatically skips the default column family handle for (int i = 0; i < tmpColumnFamilyDescriptors.size(); ++i) { ColumnFamilyHandle tmpColumnFamilyHandle = tmpColumnFamilyHandles.get(i); ColumnFamilyHandle targetColumnFamilyHandle = getOrRegisterStateColumnFamilyHandle( null, tmpRestoreDBInfo.stateMetaInfoSnapshots.get(i)) .columnFamilyHandle; try (RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(tmpRestoreDBInfo.db, tmpColumnFamilyHandle)) { iterator.seek(startKeyGroupPrefixBytes); while (iterator.isValid()) { if (RocksDBIncrementalCheckpointUtils.beforeThePrefixBytes(iterator.key(), stopKeyGroupPrefixBytes)) { writeBatchWrapper.put(targetColumnFamilyHandle, iterator.key(), iterator.value()); } else { // Since the iterator will visit the record according to the sorted order, // we can just break here. break; } iterator.next(); } } // releases native iterator resources } } finally { cleanUpPathQuietly(temporaryRestoreInstancePath); } } }
Example 19
Source File: RocksDBIncrementalRestoreOperation.java From flink with Apache License 2.0 | 4 votes |
/** * Recovery from multi incremental states with rescaling. For rescaling, this method creates a temporary * RocksDB instance for a key-groups shard. All contents from the temporary instance are copied into the * real restore instance and then the temporary instance is discarded. */ private void restoreWithRescaling(Collection<KeyedStateHandle> restoreStateHandles) throws Exception { // Prepare for restore with rescaling KeyedStateHandle initialHandle = RocksDBIncrementalCheckpointUtils.chooseTheBestStateHandleForInitial( restoreStateHandles, keyGroupRange); // Init base DB instance if (initialHandle != null) { restoreStateHandles.remove(initialHandle); initDBWithRescaling(initialHandle); } else { openDB(); } // Transfer remaining key-groups from temporary instance into base DB byte[] startKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes]; RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getStartKeyGroup(), startKeyGroupPrefixBytes); byte[] stopKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes]; RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getEndKeyGroup() + 1, stopKeyGroupPrefixBytes); for (KeyedStateHandle rawStateHandle : restoreStateHandles) { if (!(rawStateHandle instanceof IncrementalRemoteKeyedStateHandle)) { throw new IllegalStateException("Unexpected state handle type, " + "expected " + IncrementalRemoteKeyedStateHandle.class + ", but found " + rawStateHandle.getClass()); } Path temporaryRestoreInstancePath = instanceBasePath.getAbsoluteFile().toPath().resolve(UUID.randomUUID().toString()); try (RestoredDBInstance tmpRestoreDBInfo = restoreDBInstanceFromStateHandle( (IncrementalRemoteKeyedStateHandle) rawStateHandle, temporaryRestoreInstancePath); RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(this.db, writeBatchSize)) { List<ColumnFamilyDescriptor> tmpColumnFamilyDescriptors = tmpRestoreDBInfo.columnFamilyDescriptors; List<ColumnFamilyHandle> tmpColumnFamilyHandles = tmpRestoreDBInfo.columnFamilyHandles; // iterating only the requested descriptors automatically skips the default column family handle for (int i = 0; i < tmpColumnFamilyDescriptors.size(); ++i) { ColumnFamilyHandle tmpColumnFamilyHandle = tmpColumnFamilyHandles.get(i); ColumnFamilyHandle targetColumnFamilyHandle = getOrRegisterStateColumnFamilyHandle( null, tmpRestoreDBInfo.stateMetaInfoSnapshots.get(i)) .columnFamilyHandle; try (RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(tmpRestoreDBInfo.db, tmpColumnFamilyHandle, tmpRestoreDBInfo.readOptions)) { iterator.seek(startKeyGroupPrefixBytes); while (iterator.isValid()) { if (RocksDBIncrementalCheckpointUtils.beforeThePrefixBytes(iterator.key(), stopKeyGroupPrefixBytes)) { writeBatchWrapper.put(targetColumnFamilyHandle, iterator.key(), iterator.value()); } else { // Since the iterator will visit the record according to the sorted order, // we can just break here. break; } iterator.next(); } } // releases native iterator resources } } finally { cleanUpPathQuietly(temporaryRestoreInstancePath); } } }
Example 20
Source File: HeapRestoreOperation.java From flink with Apache License 2.0 | 4 votes |
@Override public Void restore() throws Exception { registeredKVStates.clear(); registeredPQStates.clear(); boolean keySerializerRestored = false; for (KeyedStateHandle keyedStateHandle : restoreStateHandles) { if (keyedStateHandle == null) { continue; } if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) { throw new IllegalStateException("Unexpected state handle type, " + "expected: " + KeyGroupsStateHandle.class + ", but found: " + keyedStateHandle.getClass()); } KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle; FSDataInputStream fsDataInputStream = keyGroupsStateHandle.openInputStream(); cancelStreamRegistry.registerCloseable(fsDataInputStream); try { DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(fsDataInputStream); KeyedBackendSerializationProxy<K> serializationProxy = new KeyedBackendSerializationProxy<>(userCodeClassLoader); serializationProxy.read(inView); if (!keySerializerRestored) { // check for key serializer compatibility; this also reconfigures the // key serializer to be compatible, if it is required and is possible TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat = keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(serializationProxy.getKeySerializerSnapshot()); if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) { throw new StateMigrationException("The new key serializer must be compatible."); } keySerializerRestored = true; } List<StateMetaInfoSnapshot> restoredMetaInfos = serializationProxy.getStateMetaInfoSnapshots(); final Map<Integer, StateMetaInfoSnapshot> kvStatesById = new HashMap<>(); createOrCheckStateForMetaInfo(restoredMetaInfos, kvStatesById); readStateHandleStateData( fsDataInputStream, inView, keyGroupsStateHandle.getGroupRangeOffsets(), kvStatesById, restoredMetaInfos.size(), serializationProxy.getReadVersion(), serializationProxy.isUsingKeyGroupCompression()); } finally { if (cancelStreamRegistry.unregisterCloseable(fsDataInputStream)) { IOUtils.closeQuietly(fsDataInputStream); } } } return null; }