Java Code Examples for org.apache.flink.runtime.state.SnapshotResult#getJobManagerOwnedSnapshot()
The following examples show how to use
org.apache.flink.runtime.state.SnapshotResult#getJobManagerOwnedSnapshot() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: RocksDBStateBackendTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testCompletingSnapshot() throws Exception {
setupRocksKeyedStateBackend();
try {
RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot =
keyedStateBackend.snapshot(0L, 0L, testStreamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
Thread asyncSnapshotThread = new Thread(snapshot);
asyncSnapshotThread.start();
waiter.await(); // wait for snapshot to run
waiter.reset();
runStateUpdates();
blocker.trigger(); // allow checkpointing to start writing
waiter.await(); // wait for snapshot stream writing to run
SnapshotResult<KeyedStateHandle> snapshotResult = snapshot.get();
KeyedStateHandle keyedStateHandle = snapshotResult.getJobManagerOwnedSnapshot();
assertNotNull(keyedStateHandle);
assertTrue(keyedStateHandle.getStateSize() > 0);
assertEquals(2, keyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
for (BlockingCheckpointOutputStream stream : testStreamFactory.getAllCreatedStreams()) {
assertTrue(stream.isClosed());
}
asyncSnapshotThread.join();
verifyRocksObjectsReleased();
} finally {
this.keyedStateBackend.dispose();
this.keyedStateBackend = null;
}
}
Example 2
| Source File: OperatorSnapshotFinalizer.java From flink with Apache License 2.0 | 5 votes |
|
public OperatorSnapshotFinalizer(
@Nonnull OperatorSnapshotFutures snapshotFutures) throws ExecutionException, InterruptedException {
SnapshotResult<KeyedStateHandle> keyedManaged =
FutureUtils.runIfNotDoneAndGet(snapshotFutures.getKeyedStateManagedFuture());
SnapshotResult<KeyedStateHandle> keyedRaw =
FutureUtils.runIfNotDoneAndGet(snapshotFutures.getKeyedStateRawFuture());
SnapshotResult<OperatorStateHandle> operatorManaged =
FutureUtils.runIfNotDoneAndGet(snapshotFutures.getOperatorStateManagedFuture());
SnapshotResult<OperatorStateHandle> operatorRaw =
FutureUtils.runIfNotDoneAndGet(snapshotFutures.getOperatorStateRawFuture());
SnapshotResult<StateObjectCollection<InputChannelStateHandle>> inputChannel = snapshotFutures.getInputChannelStateFuture().get();
SnapshotResult<StateObjectCollection<ResultSubpartitionStateHandle>> resultSubpartition = snapshotFutures.getResultSubpartitionStateFuture().get();
jobManagerOwnedState = new OperatorSubtaskState(
operatorManaged.getJobManagerOwnedSnapshot(),
operatorRaw.getJobManagerOwnedSnapshot(),
keyedManaged.getJobManagerOwnedSnapshot(),
keyedRaw.getJobManagerOwnedSnapshot(),
inputChannel.getJobManagerOwnedSnapshot(),
resultSubpartition.getJobManagerOwnedSnapshot()
);
taskLocalState = new OperatorSubtaskState(
operatorManaged.getTaskLocalSnapshot(),
operatorRaw.getTaskLocalSnapshot(),
keyedManaged.getTaskLocalSnapshot(),
keyedRaw.getTaskLocalSnapshot(),
inputChannel.getTaskLocalSnapshot(),
resultSubpartition.getTaskLocalSnapshot()
);
}
Example 3
| Source File: StateBackendTestContext.java From flink with Apache License 2.0 | 5 votes |
|
KeyedStateHandle takeSnapshot() throws Exception {
SnapshotResult<KeyedStateHandle> snapshotResult = triggerSnapshot().get();
KeyedStateHandle jobManagerOwnedSnapshot = snapshotResult.getJobManagerOwnedSnapshot();
if (jobManagerOwnedSnapshot != null) {
jobManagerOwnedSnapshot.registerSharedStates(sharedStateRegistry);
}
return jobManagerOwnedSnapshot;
}
Example 4
| Source File: RocksDBStateBackendTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testCompletingSnapshot() throws Exception {
setupRocksKeyedStateBackend();
try {
RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot =
keyedStateBackend.snapshot(0L, 0L, testStreamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
Thread asyncSnapshotThread = new Thread(snapshot);
asyncSnapshotThread.start();
waiter.await(); // wait for snapshot to run
waiter.reset();
runStateUpdates();
blocker.trigger(); // allow checkpointing to start writing
waiter.await(); // wait for snapshot stream writing to run
SnapshotResult<KeyedStateHandle> snapshotResult = snapshot.get();
KeyedStateHandle keyedStateHandle = snapshotResult.getJobManagerOwnedSnapshot();
assertNotNull(keyedStateHandle);
assertTrue(keyedStateHandle.getStateSize() > 0);
assertEquals(2, keyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
for (BlockingCheckpointOutputStream stream : testStreamFactory.getAllCreatedStreams()) {
assertTrue(stream.isClosed());
}
asyncSnapshotThread.join();
verifyRocksObjectsReleased();
} finally {
this.keyedStateBackend.dispose();
this.keyedStateBackend = null;
}
}
Example 5
| Source File: OperatorSnapshotFinalizer.java From flink with Apache License 2.0 | 5 votes |
|
public OperatorSnapshotFinalizer(
@Nonnull OperatorSnapshotFutures snapshotFutures) throws ExecutionException, InterruptedException {
SnapshotResult<KeyedStateHandle> keyedManaged =
FutureUtils.runIfNotDoneAndGet(snapshotFutures.getKeyedStateManagedFuture());
SnapshotResult<KeyedStateHandle> keyedRaw =
FutureUtils.runIfNotDoneAndGet(snapshotFutures.getKeyedStateRawFuture());
SnapshotResult<OperatorStateHandle> operatorManaged =
FutureUtils.runIfNotDoneAndGet(snapshotFutures.getOperatorStateManagedFuture());
SnapshotResult<OperatorStateHandle> operatorRaw =
FutureUtils.runIfNotDoneAndGet(snapshotFutures.getOperatorStateRawFuture());
jobManagerOwnedState = new OperatorSubtaskState(
operatorManaged.getJobManagerOwnedSnapshot(),
operatorRaw.getJobManagerOwnedSnapshot(),
keyedManaged.getJobManagerOwnedSnapshot(),
keyedRaw.getJobManagerOwnedSnapshot()
);
taskLocalState = new OperatorSubtaskState(
operatorManaged.getTaskLocalSnapshot(),
operatorRaw.getTaskLocalSnapshot(),
keyedManaged.getTaskLocalSnapshot(),
keyedRaw.getTaskLocalSnapshot()
);
}
Example 6
| Source File: StateBackendTestContext.java From flink with Apache License 2.0 | 5 votes |
|
KeyedStateHandle takeSnapshot() throws Exception {
SnapshotResult<KeyedStateHandle> snapshotResult = triggerSnapshot().get();
KeyedStateHandle jobManagerOwnedSnapshot = snapshotResult.getJobManagerOwnedSnapshot();
if (jobManagerOwnedSnapshot != null) {
jobManagerOwnedSnapshot.registerSharedStates(sharedStateRegistry);
}
return jobManagerOwnedSnapshot;
}
Example 7
| Source File: RocksDBStateBackendTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testCompletingSnapshot() throws Exception {
setupRocksKeyedStateBackend();
try {
RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot =
keyedStateBackend.snapshot(0L, 0L, testStreamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
Thread asyncSnapshotThread = new Thread(snapshot);
asyncSnapshotThread.start();
waiter.await(); // wait for snapshot to run
waiter.reset();
runStateUpdates();
blocker.trigger(); // allow checkpointing to start writing
waiter.await(); // wait for snapshot stream writing to run
SnapshotResult<KeyedStateHandle> snapshotResult = snapshot.get();
KeyedStateHandle keyedStateHandle = snapshotResult.getJobManagerOwnedSnapshot();
assertNotNull(keyedStateHandle);
assertTrue(keyedStateHandle.getStateSize() > 0);
assertEquals(2, keyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
for (BlockingCheckpointOutputStream stream : testStreamFactory.getAllCreatedStreams()) {
assertTrue(stream.isClosed());
}
asyncSnapshotThread.join();
verifyRocksObjectsReleased();
} finally {
this.keyedStateBackend.dispose();
this.keyedStateBackend = null;
}
}
Example 8
| Source File: OperatorSnapshotFinalizer.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
public OperatorSnapshotFinalizer(
@Nonnull OperatorSnapshotFutures snapshotFutures) throws ExecutionException, InterruptedException {
SnapshotResult<KeyedStateHandle> keyedManaged =
FutureUtils.runIfNotDoneAndGet(snapshotFutures.getKeyedStateManagedFuture());
SnapshotResult<KeyedStateHandle> keyedRaw =
FutureUtils.runIfNotDoneAndGet(snapshotFutures.getKeyedStateRawFuture());
SnapshotResult<OperatorStateHandle> operatorManaged =
FutureUtils.runIfNotDoneAndGet(snapshotFutures.getOperatorStateManagedFuture());
SnapshotResult<OperatorStateHandle> operatorRaw =
FutureUtils.runIfNotDoneAndGet(snapshotFutures.getOperatorStateRawFuture());
jobManagerOwnedState = new OperatorSubtaskState(
operatorManaged.getJobManagerOwnedSnapshot(),
operatorRaw.getJobManagerOwnedSnapshot(),
keyedManaged.getJobManagerOwnedSnapshot(),
keyedRaw.getJobManagerOwnedSnapshot()
);
taskLocalState = new OperatorSubtaskState(
operatorManaged.getTaskLocalSnapshot(),
operatorRaw.getTaskLocalSnapshot(),
keyedManaged.getTaskLocalSnapshot(),
keyedRaw.getTaskLocalSnapshot()
);
}
Example 9
| Source File: StateBackendTestContext.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
KeyedStateHandle takeSnapshot() throws Exception {
SnapshotResult<KeyedStateHandle> snapshotResult = triggerSnapshot().get();
KeyedStateHandle jobManagerOwnedSnapshot = snapshotResult.getJobManagerOwnedSnapshot();
if (jobManagerOwnedSnapshot != null) {
jobManagerOwnedSnapshot.registerSharedStates(sharedStateRegistry);
}
return jobManagerOwnedSnapshot;
}
Example 10
| Source File: RocksIncrementalSnapshotStrategy.java From flink with Apache License 2.0 | 4 votes |
|
@Override
protected SnapshotResult<KeyedStateHandle> callInternal() throws Exception {
boolean completed = false;
// Handle to the meta data file
SnapshotResult<StreamStateHandle> metaStateHandle = null;
// Handles to new sst files since the last completed checkpoint will go here
final Map<StateHandleID, StreamStateHandle> sstFiles = new HashMap<>();
// Handles to the misc files in the current snapshot will go here
final Map<StateHandleID, StreamStateHandle> miscFiles = new HashMap<>();
try {
metaStateHandle = materializeMetaData();
// Sanity checks - they should never fail
Preconditions.checkNotNull(metaStateHandle, "Metadata was not properly created.");
Preconditions.checkNotNull(metaStateHandle.getJobManagerOwnedSnapshot(),
"Metadata for job manager was not properly created.");
uploadSstFiles(sstFiles, miscFiles);
synchronized (materializedSstFiles) {
materializedSstFiles.put(checkpointId, sstFiles.keySet());
}
final IncrementalRemoteKeyedStateHandle jmIncrementalKeyedStateHandle =
new IncrementalRemoteKeyedStateHandle(
backendUID,
keyGroupRange,
checkpointId,
sstFiles,
miscFiles,
metaStateHandle.getJobManagerOwnedSnapshot());
final DirectoryStateHandle directoryStateHandle = localBackupDirectory.completeSnapshotAndGetHandle();
final SnapshotResult<KeyedStateHandle> snapshotResult;
if (directoryStateHandle != null && metaStateHandle.getTaskLocalSnapshot() != null) {
IncrementalLocalKeyedStateHandle localDirKeyedStateHandle =
new IncrementalLocalKeyedStateHandle(
backendUID,
checkpointId,
directoryStateHandle,
keyGroupRange,
metaStateHandle.getTaskLocalSnapshot(),
sstFiles.keySet());
snapshotResult = SnapshotResult.withLocalState(jmIncrementalKeyedStateHandle, localDirKeyedStateHandle);
} else {
snapshotResult = SnapshotResult.of(jmIncrementalKeyedStateHandle);
}
completed = true;
return snapshotResult;
} finally {
if (!completed) {
final List<StateObject> statesToDiscard =
new ArrayList<>(1 + miscFiles.size() + sstFiles.size());
statesToDiscard.add(metaStateHandle);
statesToDiscard.addAll(miscFiles.values());
statesToDiscard.addAll(sstFiles.values());
cleanupIncompleteSnapshot(statesToDiscard);
}
}
}
Example 11
| Source File: RocksDBStateBackendTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testSharedIncrementalStateDeRegistration() throws Exception {
if (enableIncrementalCheckpointing) {
AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
try {
ValueStateDescriptor<String> kvId =
new ValueStateDescriptor<>("id", String.class, null);
kvId.initializeSerializerUnlessSet(new ExecutionConfig());
ValueState<String> state =
backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
Queue<IncrementalRemoteKeyedStateHandle> previousStateHandles = new LinkedList<>();
SharedStateRegistry sharedStateRegistry = spy(new SharedStateRegistry());
for (int checkpointId = 0; checkpointId < 3; ++checkpointId) {
reset(sharedStateRegistry);
backend.setCurrentKey(checkpointId);
state.update("Hello-" + checkpointId);
RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = backend.snapshot(
checkpointId,
checkpointId,
createStreamFactory(),
CheckpointOptions.forCheckpointWithDefaultLocation());
snapshot.run();
SnapshotResult<KeyedStateHandle> snapshotResult = snapshot.get();
IncrementalRemoteKeyedStateHandle stateHandle =
(IncrementalRemoteKeyedStateHandle) snapshotResult.getJobManagerOwnedSnapshot();
Map<StateHandleID, StreamStateHandle> sharedState =
new HashMap<>(stateHandle.getSharedState());
stateHandle.registerSharedStates(sharedStateRegistry);
for (Map.Entry<StateHandleID, StreamStateHandle> e : sharedState.entrySet()) {
verify(sharedStateRegistry).registerReference(
stateHandle.createSharedStateRegistryKeyFromFileName(e.getKey()),
e.getValue());
}
previousStateHandles.add(stateHandle);
backend.notifyCheckpointComplete(checkpointId);
//-----------------------------------------------------------------
if (previousStateHandles.size() > 1) {
checkRemove(previousStateHandles.remove(), sharedStateRegistry);
}
}
while (!previousStateHandles.isEmpty()) {
reset(sharedStateRegistry);
checkRemove(previousStateHandles.remove(), sharedStateRegistry);
}
} finally {
IOUtils.closeQuietly(backend);
backend.dispose();
}
}
}
Example 12
| Source File: HeapKeyedStateBackendSnapshotMigrationTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testMapStateMigrationAfterHashMapSerRemoval() throws Exception {
ClassLoader cl = getClass().getClassLoader();
URL resource = cl.getResource("heap_keyed_statebackend_1_5_map.snapshot");
Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");
final SnapshotResult<KeyedStateHandle> stateHandles;
try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
stateHandles = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
}
final KeyedStateHandle stateHandle = stateHandles.getJobManagerOwnedSnapshot();
try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
final Integer namespace1 = 1;
final Integer namespace2 = 2;
final Integer namespace3 = 3;
final MapStateDescriptor<Long, Long> stateDescr = new MapStateDescriptor<>("my-map-state", Long.class, Long.class);
stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());
InternalMapState<String, Integer, Long, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);
keyedBackend.setCurrentKey("abc");
state.setCurrentNamespace(namespace1);
assertEquals(33L, (long) state.get(33L));
assertEquals(55L, (long) state.get(55L));
assertEquals(2, getStateSize(state));
state.setCurrentNamespace(namespace2);
assertEquals(22L, (long) state.get(22L));
assertEquals(11L, (long) state.get(11L));
assertEquals(2, getStateSize(state));
state.setCurrentNamespace(namespace3);
assertEquals(44L, (long) state.get(44L));
assertEquals(1, getStateSize(state));
keyedBackend.setCurrentKey("def");
state.setCurrentNamespace(namespace1);
assertEquals(11L, (long) state.get(11L));
assertEquals(44L, (long) state.get(44L));
assertEquals(2, getStateSize(state));
state.setCurrentNamespace(namespace3);
assertEquals(22L, (long) state.get(22L));
assertEquals(55L, (long) state.get(55L));
assertEquals(33L, (long) state.get(33L));
assertEquals(3, getStateSize(state));
keyedBackend.setCurrentKey("jkl");
state.setCurrentNamespace(namespace1);
assertEquals(11L, (long) state.get(11L));
assertEquals(22L, (long) state.get(22L));
assertEquals(33L, (long) state.get(33L));
assertEquals(44L, (long) state.get(44L));
assertEquals(55L, (long) state.get(55L));
assertEquals(5, getStateSize(state));
keyedBackend.setCurrentKey("mno");
state.setCurrentNamespace(namespace3);
assertEquals(11L, (long) state.get(11L));
assertEquals(22L, (long) state.get(22L));
assertEquals(33L, (long) state.get(33L));
assertEquals(44L, (long) state.get(44L));
assertEquals(55L, (long) state.get(55L));
assertEquals(5, getStateSize(state));
RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = keyedBackend.snapshot(
1L,
1L,
new MemCheckpointStreamFactory(4 * 1024 * 1024),
CheckpointOptions.forCheckpointWithDefaultLocation());
snapshot.run();
}
}
Example 13
| Source File: RocksIncrementalSnapshotStrategy.java From flink with Apache License 2.0 | 4 votes |
|
@Override
protected SnapshotResult<KeyedStateHandle> callInternal() throws Exception {
boolean completed = false;
// Handle to the meta data file
SnapshotResult<StreamStateHandle> metaStateHandle = null;
// Handles to new sst files since the last completed checkpoint will go here
final Map<StateHandleID, StreamStateHandle> sstFiles = new HashMap<>();
// Handles to the misc files in the current snapshot will go here
final Map<StateHandleID, StreamStateHandle> miscFiles = new HashMap<>();
try {
metaStateHandle = materializeMetaData();
// Sanity checks - they should never fail
Preconditions.checkNotNull(metaStateHandle, "Metadata was not properly created.");
Preconditions.checkNotNull(metaStateHandle.getJobManagerOwnedSnapshot(),
"Metadata for job manager was not properly created.");
uploadSstFiles(sstFiles, miscFiles);
synchronized (materializedSstFiles) {
materializedSstFiles.put(checkpointId, sstFiles.keySet());
}
final IncrementalRemoteKeyedStateHandle jmIncrementalKeyedStateHandle =
new IncrementalRemoteKeyedStateHandle(
backendUID,
keyGroupRange,
checkpointId,
sstFiles,
miscFiles,
metaStateHandle.getJobManagerOwnedSnapshot());
final DirectoryStateHandle directoryStateHandle = localBackupDirectory.completeSnapshotAndGetHandle();
final SnapshotResult<KeyedStateHandle> snapshotResult;
if (directoryStateHandle != null && metaStateHandle.getTaskLocalSnapshot() != null) {
IncrementalLocalKeyedStateHandle localDirKeyedStateHandle =
new IncrementalLocalKeyedStateHandle(
backendUID,
checkpointId,
directoryStateHandle,
keyGroupRange,
metaStateHandle.getTaskLocalSnapshot(),
sstFiles.keySet());
snapshotResult = SnapshotResult.withLocalState(jmIncrementalKeyedStateHandle, localDirKeyedStateHandle);
} else {
snapshotResult = SnapshotResult.of(jmIncrementalKeyedStateHandle);
}
completed = true;
return snapshotResult;
} finally {
if (!completed) {
final List<StateObject> statesToDiscard =
new ArrayList<>(1 + miscFiles.size() + sstFiles.size());
statesToDiscard.add(metaStateHandle);
statesToDiscard.addAll(miscFiles.values());
statesToDiscard.addAll(sstFiles.values());
cleanupIncompleteSnapshot(statesToDiscard);
}
}
}
Example 14
| Source File: HeapKeyedStateBackendSnapshotMigrationTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
public void testMapStateMigrationAfterHashMapSerRemoval() throws Exception {
ClassLoader cl = getClass().getClassLoader();
URL resource = cl.getResource("heap_keyed_statebackend_1_5_map.snapshot");
Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");
final SnapshotResult<KeyedStateHandle> stateHandles;
try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
stateHandles = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
}
final KeyedStateHandle stateHandle = stateHandles.getJobManagerOwnedSnapshot();
try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
final Integer namespace1 = 1;
final Integer namespace2 = 2;
final Integer namespace3 = 3;
final MapStateDescriptor<Long, Long> stateDescr = new MapStateDescriptor<>("my-map-state", Long.class, Long.class);
stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());
InternalMapState<String, Integer, Long, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);
keyedBackend.setCurrentKey("abc");
state.setCurrentNamespace(namespace1);
assertEquals(33L, (long) state.get(33L));
assertEquals(55L, (long) state.get(55L));
assertEquals(2, getStateSize(state));
state.setCurrentNamespace(namespace2);
assertEquals(22L, (long) state.get(22L));
assertEquals(11L, (long) state.get(11L));
assertEquals(2, getStateSize(state));
state.setCurrentNamespace(namespace3);
assertEquals(44L, (long) state.get(44L));
assertEquals(1, getStateSize(state));
keyedBackend.setCurrentKey("def");
state.setCurrentNamespace(namespace1);
assertEquals(11L, (long) state.get(11L));
assertEquals(44L, (long) state.get(44L));
assertEquals(2, getStateSize(state));
state.setCurrentNamespace(namespace3);
assertEquals(22L, (long) state.get(22L));
assertEquals(55L, (long) state.get(55L));
assertEquals(33L, (long) state.get(33L));
assertEquals(3, getStateSize(state));
keyedBackend.setCurrentKey("jkl");
state.setCurrentNamespace(namespace1);
assertEquals(11L, (long) state.get(11L));
assertEquals(22L, (long) state.get(22L));
assertEquals(33L, (long) state.get(33L));
assertEquals(44L, (long) state.get(44L));
assertEquals(55L, (long) state.get(55L));
assertEquals(5, getStateSize(state));
keyedBackend.setCurrentKey("mno");
state.setCurrentNamespace(namespace3);
assertEquals(11L, (long) state.get(11L));
assertEquals(22L, (long) state.get(22L));
assertEquals(33L, (long) state.get(33L));
assertEquals(44L, (long) state.get(44L));
assertEquals(55L, (long) state.get(55L));
assertEquals(5, getStateSize(state));
RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = keyedBackend.snapshot(
1L,
1L,
new MemCheckpointStreamFactory(4 * 1024 * 1024),
CheckpointOptions.forCheckpointWithDefaultLocation());
snapshot.run();
}
}
Example 15
| Source File: RocksDBStateBackendTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testSharedIncrementalStateDeRegistration() throws Exception {
if (enableIncrementalCheckpointing) {
AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
try {
ValueStateDescriptor<String> kvId =
new ValueStateDescriptor<>("id", String.class, null);
kvId.initializeSerializerUnlessSet(new ExecutionConfig());
ValueState<String> state =
backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
Queue<IncrementalRemoteKeyedStateHandle> previousStateHandles = new LinkedList<>();
SharedStateRegistry sharedStateRegistry = spy(new SharedStateRegistry());
for (int checkpointId = 0; checkpointId < 3; ++checkpointId) {
reset(sharedStateRegistry);
backend.setCurrentKey(checkpointId);
state.update("Hello-" + checkpointId);
RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = backend.snapshot(
checkpointId,
checkpointId,
createStreamFactory(),
CheckpointOptions.forCheckpointWithDefaultLocation());
snapshot.run();
SnapshotResult<KeyedStateHandle> snapshotResult = snapshot.get();
IncrementalRemoteKeyedStateHandle stateHandle =
(IncrementalRemoteKeyedStateHandle) snapshotResult.getJobManagerOwnedSnapshot();
Map<StateHandleID, StreamStateHandle> sharedState =
new HashMap<>(stateHandle.getSharedState());
stateHandle.registerSharedStates(sharedStateRegistry);
for (Map.Entry<StateHandleID, StreamStateHandle> e : sharedState.entrySet()) {
verify(sharedStateRegistry).registerReference(
stateHandle.createSharedStateRegistryKeyFromFileName(e.getKey()),
e.getValue());
}
previousStateHandles.add(stateHandle);
backend.notifyCheckpointComplete(checkpointId);
//-----------------------------------------------------------------
if (previousStateHandles.size() > 1) {
checkRemove(previousStateHandles.remove(), sharedStateRegistry);
}
}
while (!previousStateHandles.isEmpty()) {
reset(sharedStateRegistry);
checkRemove(previousStateHandles.remove(), sharedStateRegistry);
}
} finally {
IOUtils.closeQuietly(backend);
backend.dispose();
}
}
}
Example 16
| Source File: HeapKeyedStateBackendSnapshotMigrationTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testMapStateMigrationAfterHashMapSerRemoval() throws Exception {
ClassLoader cl = getClass().getClassLoader();
URL resource = cl.getResource("heap_keyed_statebackend_1_5_map.snapshot");
Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");
final SnapshotResult<KeyedStateHandle> stateHandles;
try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
stateHandles = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
}
final KeyedStateHandle stateHandle = stateHandles.getJobManagerOwnedSnapshot();
try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
final Integer namespace1 = 1;
final Integer namespace2 = 2;
final Integer namespace3 = 3;
final MapStateDescriptor<Long, Long> stateDescr = new MapStateDescriptor<>("my-map-state", Long.class, Long.class);
stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());
InternalMapState<String, Integer, Long, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);
keyedBackend.setCurrentKey("abc");
state.setCurrentNamespace(namespace1);
assertEquals(33L, (long) state.get(33L));
assertEquals(55L, (long) state.get(55L));
assertEquals(2, getStateSize(state));
state.setCurrentNamespace(namespace2);
assertEquals(22L, (long) state.get(22L));
assertEquals(11L, (long) state.get(11L));
assertEquals(2, getStateSize(state));
state.setCurrentNamespace(namespace3);
assertEquals(44L, (long) state.get(44L));
assertEquals(1, getStateSize(state));
keyedBackend.setCurrentKey("def");
state.setCurrentNamespace(namespace1);
assertEquals(11L, (long) state.get(11L));
assertEquals(44L, (long) state.get(44L));
assertEquals(2, getStateSize(state));
state.setCurrentNamespace(namespace3);
assertEquals(22L, (long) state.get(22L));
assertEquals(55L, (long) state.get(55L));
assertEquals(33L, (long) state.get(33L));
assertEquals(3, getStateSize(state));
keyedBackend.setCurrentKey("jkl");
state.setCurrentNamespace(namespace1);
assertEquals(11L, (long) state.get(11L));
assertEquals(22L, (long) state.get(22L));
assertEquals(33L, (long) state.get(33L));
assertEquals(44L, (long) state.get(44L));
assertEquals(55L, (long) state.get(55L));
assertEquals(5, getStateSize(state));
keyedBackend.setCurrentKey("mno");
state.setCurrentNamespace(namespace3);
assertEquals(11L, (long) state.get(11L));
assertEquals(22L, (long) state.get(22L));
assertEquals(33L, (long) state.get(33L));
assertEquals(44L, (long) state.get(44L));
assertEquals(55L, (long) state.get(55L));
assertEquals(5, getStateSize(state));
RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = keyedBackend.snapshot(
1L,
1L,
new MemCheckpointStreamFactory(4 * 1024 * 1024),
CheckpointOptions.forCheckpointWithDefaultLocation());
snapshot.run();
}
}
Example 17
| Source File: RocksDBStateBackendTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
public void testSharedIncrementalStateDeRegistration() throws Exception {
if (enableIncrementalCheckpointing) {
AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
try {
ValueStateDescriptor<String> kvId =
new ValueStateDescriptor<>("id", String.class, null);
kvId.initializeSerializerUnlessSet(new ExecutionConfig());
ValueState<String> state =
backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
Queue<IncrementalRemoteKeyedStateHandle> previousStateHandles = new LinkedList<>();
SharedStateRegistry sharedStateRegistry = spy(new SharedStateRegistry());
for (int checkpointId = 0; checkpointId < 3; ++checkpointId) {
reset(sharedStateRegistry);
backend.setCurrentKey(checkpointId);
state.update("Hello-" + checkpointId);
RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = backend.snapshot(
checkpointId,
checkpointId,
createStreamFactory(),
CheckpointOptions.forCheckpointWithDefaultLocation());
snapshot.run();
SnapshotResult<KeyedStateHandle> snapshotResult = snapshot.get();
IncrementalRemoteKeyedStateHandle stateHandle =
(IncrementalRemoteKeyedStateHandle) snapshotResult.getJobManagerOwnedSnapshot();
Map<StateHandleID, StreamStateHandle> sharedState =
new HashMap<>(stateHandle.getSharedState());
stateHandle.registerSharedStates(sharedStateRegistry);
for (Map.Entry<StateHandleID, StreamStateHandle> e : sharedState.entrySet()) {
verify(sharedStateRegistry).registerReference(
stateHandle.createSharedStateRegistryKeyFromFileName(e.getKey()),
e.getValue());
}
previousStateHandles.add(stateHandle);
backend.notifyCheckpointComplete(checkpointId);
//-----------------------------------------------------------------
if (previousStateHandles.size() > 1) {
checkRemove(previousStateHandles.remove(), sharedStateRegistry);
}
}
while (!previousStateHandles.isEmpty()) {
reset(sharedStateRegistry);
checkRemove(previousStateHandles.remove(), sharedStateRegistry);
}
} finally {
IOUtils.closeQuietly(backend);
backend.dispose();
}
}
}
Example 18
| Source File: RocksIncrementalSnapshotStrategy.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Override
protected SnapshotResult<KeyedStateHandle> callInternal() throws Exception {
boolean completed = false;
// Handle to the meta data file
SnapshotResult<StreamStateHandle> metaStateHandle = null;
// Handles to new sst files since the last completed checkpoint will go here
final Map<StateHandleID, StreamStateHandle> sstFiles = new HashMap<>();
// Handles to the misc files in the current snapshot will go here
final Map<StateHandleID, StreamStateHandle> miscFiles = new HashMap<>();
try {
metaStateHandle = materializeMetaData();
// Sanity checks - they should never fail
Preconditions.checkNotNull(metaStateHandle, "Metadata was not properly created.");
Preconditions.checkNotNull(metaStateHandle.getJobManagerOwnedSnapshot(),
"Metadata for job manager was not properly created.");
uploadSstFiles(sstFiles, miscFiles);
synchronized (materializedSstFiles) {
materializedSstFiles.put(checkpointId, sstFiles.keySet());
}
final IncrementalRemoteKeyedStateHandle jmIncrementalKeyedStateHandle =
new IncrementalRemoteKeyedStateHandle(
backendUID,
keyGroupRange,
checkpointId,
sstFiles,
miscFiles,
metaStateHandle.getJobManagerOwnedSnapshot());
final DirectoryStateHandle directoryStateHandle = localBackupDirectory.completeSnapshotAndGetHandle();
final SnapshotResult<KeyedStateHandle> snapshotResult;
if (directoryStateHandle != null && metaStateHandle.getTaskLocalSnapshot() != null) {
IncrementalLocalKeyedStateHandle localDirKeyedStateHandle =
new IncrementalLocalKeyedStateHandle(
backendUID,
checkpointId,
directoryStateHandle,
keyGroupRange,
metaStateHandle.getTaskLocalSnapshot(),
sstFiles.keySet());
snapshotResult = SnapshotResult.withLocalState(jmIncrementalKeyedStateHandle, localDirKeyedStateHandle);
} else {
snapshotResult = SnapshotResult.of(jmIncrementalKeyedStateHandle);
}
completed = true;
return snapshotResult;
} finally {
if (!completed) {
final List<StateObject> statesToDiscard =
new ArrayList<>(1 + miscFiles.size() + sstFiles.size());
statesToDiscard.add(metaStateHandle);
statesToDiscard.addAll(miscFiles.values());
statesToDiscard.addAll(sstFiles.values());
cleanupIncompleteSnapshot(statesToDiscard);
}
}
}
