org.apache.flink.runtime.state.CheckpointStorageLocationReference Java Examples
The following examples show how to use
org.apache.flink.runtime.state.CheckpointStorageLocationReference.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SubtaskCheckpointCoordinatorTest.java From flink with Apache License 2.0 | 6 votes |
private boolean initCheckpoint(boolean unalignedCheckpointEnabled, CheckpointType checkpointType) throws IOException { class MockWriter extends ChannelStateWriterImpl.NoOpChannelStateWriter { private boolean started; @Override public void start(long checkpointId, CheckpointOptions checkpointOptions) { started = true; } } MockWriter writer = new MockWriter(); SubtaskCheckpointCoordinator coordinator = coordinator(unalignedCheckpointEnabled, writer); CheckpointStorageLocationReference locationReference = CheckpointStorageLocationReference.getDefault(); CheckpointOptions options = new CheckpointOptions(checkpointType, locationReference, true, unalignedCheckpointEnabled); coordinator.initCheckpoint(1L, options); return writer.started; }
Example #2
Source File: FsCheckpointStorage.java From flink with Apache License 2.0 | 6 votes |
@Override public CheckpointStorageLocation initializeLocationForCheckpoint(long checkpointId) throws IOException { checkArgument(checkpointId >= 0, "Illegal negative checkpoint id: %d.", checkpointId); checkArgument(baseLocationsInitialized, "The base checkpoint location has not been initialized."); // prepare all the paths needed for the checkpoints final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId); // create the checkpoint exclusive directory fileSystem.mkdirs(checkpointDir); return new FsCheckpointStorageLocation( fileSystem, checkpointDir, sharedStateDirectory, taskOwnedStateDirectory, CheckpointStorageLocationReference.getDefault(), fileSizeThreshold, writeBufferSize); }
Example #3
Source File: FsCheckpointStorageLocation.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
public FsCheckpointStorageLocation( FileSystem fileSystem, Path checkpointDir, Path sharedStateDir, Path taskOwnedStateDir, CheckpointStorageLocationReference reference, int fileStateSizeThreshold) { super(fileSystem, checkpointDir, sharedStateDir, fileStateSizeThreshold); checkArgument(fileStateSizeThreshold >= 0); this.fileSystem = checkNotNull(fileSystem); this.checkpointDirectory = checkNotNull(checkpointDir); this.sharedStateDirectory = checkNotNull(sharedStateDir); this.taskOwnedStateDirectory = checkNotNull(taskOwnedStateDir); this.reference = checkNotNull(reference); // the metadata file should not have entropy in its path Path metadataDir = EntropyInjector.removeEntropyMarkerIfPresent(fileSystem, checkpointDir); this.metadataFilePath = new Path(metadataDir, AbstractFsCheckpointStorage.METADATA_FILE_NAME); this.fileStateSizeThreshold = fileStateSizeThreshold; }
Example #4
Source File: FsCheckpointStorage.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override public CheckpointStorageLocation initializeLocationForCheckpoint(long checkpointId) throws IOException { checkArgument(checkpointId >= 0); // prepare all the paths needed for the checkpoints final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId); // create the checkpoint exclusive directory fileSystem.mkdirs(checkpointDir); return new FsCheckpointStorageLocation( fileSystem, checkpointDir, sharedStateDirectory, taskOwnedStateDirectory, CheckpointStorageLocationReference.getDefault(), fileSizeThreshold); }
Example #5
Source File: FsCheckpointStorageTest.java From flink with Apache License 2.0 | 6 votes |
/** * This test checks that the expected mkdirs action for checkpoint storage, * only be called when initializeBaseLocations and not called when resolveCheckpointStorageLocation. */ @Test public void testStorageLocationMkdirs() throws Exception { FsCheckpointStorage storage = new FsCheckpointStorage( randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE); File baseDir = new File(storage.getCheckpointsDirectory().getPath()); assertFalse(baseDir.exists()); // mkdirs would only be called when initializeBaseLocations storage.initializeBaseLocations(); assertTrue(baseDir.exists()); // mkdir would not be called when resolveCheckpointStorageLocation storage = new FsCheckpointStorage( randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE); FsCheckpointStorageLocation location = (FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(177, CheckpointStorageLocationReference.getDefault()); Path checkpointPath = location.getCheckpointDirectory(); File checkpointDir = new File(checkpointPath.getPath()); assertFalse(checkpointDir.exists()); }
Example #6
Source File: PendingCheckpointTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private PendingCheckpoint createPendingCheckpoint(CheckpointProperties props, Executor executor) throws IOException { final Path checkpointDir = new Path(tmpFolder.newFolder().toURI()); final FsCheckpointStorageLocation location = new FsCheckpointStorageLocation( LocalFileSystem.getSharedInstance(), checkpointDir, checkpointDir, checkpointDir, CheckpointStorageLocationReference.getDefault(), 1024); final Map<ExecutionAttemptID, ExecutionVertex> ackTasks = new HashMap<>(ACK_TASKS); return new PendingCheckpoint( new JobID(), 0, 1, ackTasks, props, location, executor); }
Example #7
Source File: AlternatingCheckpointBarrierHandlerTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testPreviousHandlerReset() throws Exception { SingleInputGate inputGate = new SingleInputGateBuilder().setNumberOfChannels(2).build(); inputGate.setInputChannels(new TestInputChannel(inputGate, 0), new TestInputChannel(inputGate, 1)); TestInvokable target = new TestInvokable(); CheckpointBarrierAligner alignedHandler = new CheckpointBarrierAligner("test", target, inputGate); CheckpointBarrierUnaligner unalignedHandler = new CheckpointBarrierUnaligner(TestSubtaskCheckpointCoordinator.INSTANCE, "test", target, inputGate); AlternatingCheckpointBarrierHandler barrierHandler = new AlternatingCheckpointBarrierHandler(alignedHandler, unalignedHandler, target); for (int i = 0; i < 4; i++) { int channel = i % 2; CheckpointType type = channel == 0 ? CHECKPOINT : SAVEPOINT; barrierHandler.processBarrier(new CheckpointBarrier(i, 0, new CheckpointOptions(type, CheckpointStorageLocationReference.getDefault())), new InputChannelInfo(0, channel)); assertEquals(type.isSavepoint(), alignedHandler.isCheckpointPending()); assertNotEquals(alignedHandler.isCheckpointPending(), unalignedHandler.isCheckpointPending()); if (!type.isSavepoint()) { assertFalse(barrierHandler.getAllBarriersReceivedFuture(i).isDone()); assertInflightDataEquals(unalignedHandler, barrierHandler, i, inputGate.getNumberOfInputChannels()); } } }
Example #8
Source File: FsCheckpointStorageTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * This test checks that no mkdirs is called by the checkpoint storage location when resolved. */ @Test public void testStorageLocationDoesNotMkdirs() throws Exception { FsCheckpointStorage storage = new FsCheckpointStorage( randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD); File baseDir = new File(storage.getCheckpointsDirectory().getPath()); assertTrue(baseDir.exists()); FsCheckpointStorageLocation location = (FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(177, CheckpointStorageLocationReference.getDefault()); Path checkpointPath = location.getCheckpointDirectory(); File checkpointDir = new File(checkpointPath.getPath()); assertFalse(checkpointDir.exists()); }
Example #9
Source File: FsStorageLocationReferenceTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testEncodeAndDecode() throws Exception { final Path path = randomPath(new Random()); try { CheckpointStorageLocationReference ref = encodePathAsReference(path); Path decoded = decodePathFromReference(ref); assertEquals(path, decoded); } catch (Exception | Error e) { // if something goes wrong, help by printing the problematic path log.error("ERROR FOR PATH " + path); throw e; } }
Example #10
Source File: FsCheckpointStorageTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testResolveCheckpointStorageLocation() throws Exception { final FileSystem checkpointFileSystem = mock(FileSystem.class); final FsCheckpointStorage storage = new FsCheckpointStorage( new TestingPath("hdfs:///checkpoint/", checkpointFileSystem), null, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE); final FsCheckpointStorageLocation checkpointStreamFactory = (FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(1L, CheckpointStorageLocationReference.getDefault()); assertEquals(checkpointFileSystem, checkpointStreamFactory.getFileSystem()); final CheckpointStorageLocationReference savepointLocationReference = AbstractFsCheckpointStorage.encodePathAsReference(new Path("file:///savepoint/")); final FsCheckpointStorageLocation savepointStreamFactory = (FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(2L, savepointLocationReference); final FileSystem fileSystem = savepointStreamFactory.getFileSystem(); assertTrue(fileSystem instanceof LocalFileSystem); }
Example #11
Source File: FsCheckpointStorageTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testResolveCheckpointStorageLocation() throws Exception { final FileSystem checkpointFileSystem = mock(FileSystem.class); final FsCheckpointStorage storage = new FsCheckpointStorage( new TestingPath("hdfs:///checkpoint/", checkpointFileSystem), null, new JobID(), FILE_SIZE_THRESHOLD); final FsCheckpointStorageLocation checkpointStreamFactory = (FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(1L, CheckpointStorageLocationReference.getDefault()); assertEquals(checkpointFileSystem, checkpointStreamFactory.getFileSystem()); final CheckpointStorageLocationReference savepointLocationReference = AbstractFsCheckpointStorage.encodePathAsReference(new Path("file:///savepoint/")); final FsCheckpointStorageLocation savepointStreamFactory = (FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(2L, savepointLocationReference); final FileSystem fileSystem = savepointStreamFactory.getFileSystem(); assertTrue(fileSystem instanceof LocalFileSystem); }
Example #12
Source File: PendingCheckpointTest.java From flink with Apache License 2.0 | 6 votes |
private PendingCheckpoint createPendingCheckpoint(CheckpointProperties props, Executor executor) throws IOException { final Path checkpointDir = new Path(tmpFolder.newFolder().toURI()); final FsCheckpointStorageLocation location = new FsCheckpointStorageLocation( LocalFileSystem.getSharedInstance(), checkpointDir, checkpointDir, checkpointDir, CheckpointStorageLocationReference.getDefault(), 1024, 4096); final Map<ExecutionAttemptID, ExecutionVertex> ackTasks = new HashMap<>(ACK_TASKS); return new PendingCheckpoint( new JobID(), 0, 1, ackTasks, props, location, executor); }
Example #13
Source File: FsCheckpointStorage.java From flink with Apache License 2.0 | 6 votes |
@Override public CheckpointStorageLocation initializeLocationForCheckpoint(long checkpointId) throws IOException { checkArgument(checkpointId >= 0); // prepare all the paths needed for the checkpoints final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId); // create the checkpoint exclusive directory fileSystem.mkdirs(checkpointDir); return new FsCheckpointStorageLocation( fileSystem, checkpointDir, sharedStateDirectory, taskOwnedStateDirectory, CheckpointStorageLocationReference.getDefault(), fileSizeThreshold, writeBufferSize); }
Example #14
Source File: FsCheckpointStorageTest.java From flink with Apache License 2.0 | 6 votes |
/** * This test checks that no mkdirs is called by the checkpoint storage location when resolved. */ @Test public void testStorageLocationDoesNotMkdirs() throws Exception { FsCheckpointStorage storage = new FsCheckpointStorage( randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE); File baseDir = new File(storage.getCheckpointsDirectory().getPath()); assertTrue(baseDir.exists()); FsCheckpointStorageLocation location = (FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(177, CheckpointStorageLocationReference.getDefault()); Path checkpointPath = location.getCheckpointDirectory(); File checkpointDir = new File(checkpointPath.getPath()); assertFalse(checkpointDir.exists()); }
Example #15
Source File: FsCheckpointStorageTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testResolveCheckpointStorageLocation() throws Exception { final FileSystem checkpointFileSystem = mock(FileSystem.class); final FsCheckpointStorage storage = new FsCheckpointStorage( new TestingPath("hdfs:///checkpoint/", checkpointFileSystem), null, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE); final FsCheckpointStorageLocation checkpointStreamFactory = (FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(1L, CheckpointStorageLocationReference.getDefault()); assertEquals(checkpointFileSystem, checkpointStreamFactory.getFileSystem()); final CheckpointStorageLocationReference savepointLocationReference = AbstractFsCheckpointStorage.encodePathAsReference(new Path("file:///savepoint/")); final FsCheckpointStorageLocation savepointStreamFactory = (FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(2L, savepointLocationReference); final FileSystem fileSystem = savepointStreamFactory.getFileSystem(); assertTrue(fileSystem instanceof LocalFileSystem); }
Example #16
Source File: FsStorageLocationReferenceTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testEncodeAndDecode() throws Exception { final Path path = randomPath(new Random()); try { CheckpointStorageLocationReference ref = encodePathAsReference(path); Path decoded = decodePathFromReference(ref); assertEquals(path, decoded); } catch (Exception | Error e) { // if something goes wrong, help by printing the problematic path log.error("ERROR FOR PATH " + path); throw e; } }
Example #17
Source File: AlternatingCheckpointBarrierHandlerTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testOutOfOrderBarrier() throws Exception { SingleInputGate inputGate = new SingleInputGateBuilder().setNumberOfChannels(2).build(); inputGate.setInputChannels(new TestInputChannel(inputGate, 0), new TestInputChannel(inputGate, 1)); TestInvokable target = new TestInvokable(); CheckpointBarrierAligner alignedHandler = new CheckpointBarrierAligner("test", target, inputGate); CheckpointBarrierUnaligner unalignedHandler = new CheckpointBarrierUnaligner(TestSubtaskCheckpointCoordinator.INSTANCE, "test", target, inputGate); AlternatingCheckpointBarrierHandler barrierHandler = new AlternatingCheckpointBarrierHandler(alignedHandler, unalignedHandler, target); long checkpointId = 10; long outOfOrderSavepointId = 5; long initialAlignedCheckpointId = alignedHandler.getLatestCheckpointId(); barrierHandler.processBarrier(new CheckpointBarrier(checkpointId, 0, new CheckpointOptions(CHECKPOINT, CheckpointStorageLocationReference.getDefault())), new InputChannelInfo(0, 0)); barrierHandler.processBarrier(new CheckpointBarrier(outOfOrderSavepointId, 0, new CheckpointOptions(SAVEPOINT, CheckpointStorageLocationReference.getDefault())), new InputChannelInfo(0, 1)); assertEquals(checkpointId, barrierHandler.getLatestCheckpointId()); assertInflightDataEquals(unalignedHandler, barrierHandler, checkpointId, inputGate.getNumberOfInputChannels()); assertEquals(initialAlignedCheckpointId, alignedHandler.getLatestCheckpointId()); }
Example #18
Source File: SubtaskCheckpointCoordinatorTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testSkipChannelStateForSavepoints() throws Exception { SubtaskCheckpointCoordinator coordinator = new MockSubtaskCheckpointCoordinatorBuilder() .setUnalignedCheckpointEnabled(true) .setPrepareInputSnapshot((u1, u2) -> { fail("should not prepare input snapshot for savepoint"); return null; }).build(); coordinator.checkpointState( new CheckpointMetaData(0, 0), new CheckpointOptions(SAVEPOINT, CheckpointStorageLocationReference.getDefault()), new CheckpointMetrics(), new OperatorChain<>(new NoOpStreamTask<>(new DummyEnvironment()), new NonRecordWriter<>()), () -> false); }
Example #19
Source File: CheckpointOptionsTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testSavepointNeedsAlignment() { CheckpointStorageLocationReference location = CheckpointStorageLocationReference.getDefault(); assertTrue(new CheckpointOptions(SAVEPOINT, location, true, true).needsAlignment()); assertFalse(new CheckpointOptions(SAVEPOINT, location, false, true).needsAlignment()); assertTrue(new CheckpointOptions(SAVEPOINT, location, true, false).needsAlignment()); assertFalse(new CheckpointOptions(SAVEPOINT, location, false, false).needsAlignment()); }
Example #20
Source File: RemoteInputChannelTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testNoNotifyOnSavepoint() throws IOException { TestBufferReceivedListener listener = new TestBufferReceivedListener(); SingleInputGate inputGate = new SingleInputGateBuilder().build(); inputGate.registerBufferReceivedListener(listener); RemoteInputChannel channel = InputChannelBuilder.newBuilder().buildRemoteChannel(inputGate); channel.onBuffer(toBuffer(new CheckpointBarrier(123L, 123L, new CheckpointOptions(SAVEPOINT, CheckpointStorageLocationReference.getDefault()))), 0, 0); channel.checkError(); assertTrue(listener.notifiedOnBarriers.isEmpty()); }
Example #21
Source File: CheckpointOptionsTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testSavepoint() throws Exception { final Random rnd = new Random(); final byte[] locationBytes = new byte[rnd.nextInt(41) + 1]; rnd.nextBytes(locationBytes); final CheckpointOptions options = new CheckpointOptions( CheckpointType.values()[rnd.nextInt(CheckpointType.values().length)], new CheckpointStorageLocationReference(locationBytes)); final CheckpointOptions copy = CommonTestUtils.createCopySerializable(options); assertEquals(options.getCheckpointType(), copy.getCheckpointType()); assertArrayEquals(locationBytes, copy.getTargetLocation().getReferenceBytes()); }
Example #22
Source File: MemoryBackendCheckpointStorage.java From flink with Apache License 2.0 | 5 votes |
@Override public CheckpointStreamFactory resolveCheckpointStorageLocation( long checkpointId, CheckpointStorageLocationReference reference) { // no matter where the checkpoint goes, we always return the storage location that stores // state inline with the state handles. return new MemCheckpointStreamFactory(maxStateSize); }
Example #23
Source File: LocalInputChannelTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testNoNotifyOnSavepoint() throws IOException { TestBufferReceivedListener listener = new TestBufferReceivedListener(); SingleInputGate inputGate = new SingleInputGateBuilder().build(); inputGate.registerBufferReceivedListener(listener); LocalInputChannel channel = InputChannelBuilder.newBuilder().buildLocalChannel(inputGate); CheckpointBarrier barrier = new CheckpointBarrier(123L, 123L, new CheckpointOptions(SAVEPOINT, CheckpointStorageLocationReference.getDefault())); channel.notifyPriorityEvent(new BufferConsumer(toBuffer(barrier).getMemorySegment(), FreeingBufferRecycler.INSTANCE, getDataType(barrier))); channel.checkError(); assertTrue(listener.notifiedOnBarriers.isEmpty()); }
Example #24
Source File: FsCheckpointStorage.java From flink with Apache License 2.0 | 5 votes |
@Override public CheckpointStreamFactory resolveCheckpointStorageLocation( long checkpointId, CheckpointStorageLocationReference reference) throws IOException { if (reference.isDefaultReference()) { // default reference, construct the default location for that particular checkpoint final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId); return new FsCheckpointStorageLocation( fileSystem, checkpointDir, sharedStateDirectory, taskOwnedStateDirectory, reference, fileSizeThreshold, writeBufferSize); } else { // location encoded in the reference final Path path = decodePathFromReference(reference); return new FsCheckpointStorageLocation( path.getFileSystem(), path, path, path, reference, fileSizeThreshold, writeBufferSize); } }
Example #25
Source File: SylphFsCheckpointStorage.java From sylph with Apache License 2.0 | 5 votes |
@Override public CheckpointStreamFactory resolveCheckpointStorageLocation( long checkpointId, CheckpointStorageLocationReference reference) throws IOException { if (reference.isDefaultReference()) { // default reference, construct the default location for that particular checkpoint final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId); return new FsCheckpointStorageLocation( fileSystem, checkpointDir, sharedStateDirectory, taskOwnedStateDirectory, reference, fileSizeThreshold, writeBufferSize); } else { // location encoded in the reference final Path path = decodePathFromReference(reference); return new FsCheckpointStorageLocation( path.getFileSystem(), path, path, path, reference, fileSizeThreshold, writeBufferSize); } }
Example #26
Source File: SavepointOutputFormat.java From flink with Apache License 2.0 | 5 votes |
private static CheckpointStorageLocation createSavepointLocation(Path location) throws IOException { final CheckpointStorageLocationReference reference = AbstractFsCheckpointStorage.encodePathAsReference(location); return new FsCheckpointStorageLocation( location.getFileSystem(), location, location, location, reference, (int) CheckpointingOptions.FS_SMALL_FILE_THRESHOLD.defaultValue().getBytes(), CheckpointingOptions.FS_WRITE_BUFFER_SIZE.defaultValue()); }
Example #27
Source File: CheckpointOptionsTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testCheckpointNeedsAlignment() { CheckpointStorageLocationReference location = CheckpointStorageLocationReference.getDefault(); assertFalse(new CheckpointOptions(CHECKPOINT, location, true, true).needsAlignment()); assertTrue(new CheckpointOptions(CHECKPOINT, location, true, false).needsAlignment()); assertFalse(new CheckpointOptions(CHECKPOINT, location, false, true).needsAlignment()); assertFalse(new CheckpointOptions(CHECKPOINT, location, false, false).needsAlignment()); }
Example #28
Source File: AbstractFsCheckpointStorage.java From flink with Apache License 2.0 | 5 votes |
/** * Encodes the given path as a reference in bytes. The path is encoded as a UTF-8 string * and prepended as a magic number. * * @param path The path to encode. * @return The location reference. */ public static CheckpointStorageLocationReference encodePathAsReference(Path path) { byte[] refBytes = path.toString().getBytes(StandardCharsets.UTF_8); byte[] bytes = new byte[REFERENCE_MAGIC_NUMBER.length + refBytes.length]; System.arraycopy(REFERENCE_MAGIC_NUMBER, 0, bytes, 0, REFERENCE_MAGIC_NUMBER.length); System.arraycopy(refBytes, 0, bytes, REFERENCE_MAGIC_NUMBER.length, refBytes.length); return new CheckpointStorageLocationReference(bytes); }
Example #29
Source File: SylphFsCheckpointStorage.java From sylph with Apache License 2.0 | 5 votes |
@Override protected CheckpointStorageLocation createSavepointLocation(FileSystem fs, Path location) throws IOException { final CheckpointStorageLocationReference reference = encodePathAsReference(location); return new FsCheckpointStorageLocation(fs, location, location, location, reference, fileSizeThreshold, writeBufferSize); }
Example #30
Source File: PipelinedSubpartitionWithReadViewTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testBarrierOvertaking() throws Exception { subpartition.add(createFilledFinishedBufferConsumer(1)); assertEquals(0, availablityListener.getNumNotifications()); assertEquals(0, availablityListener.getNumPriorityEvents()); subpartition.add(createFilledFinishedBufferConsumer(2)); assertEquals(1, availablityListener.getNumNotifications()); assertEquals(0, availablityListener.getNumPriorityEvents()); BufferConsumer eventBuffer = EventSerializer.toBufferConsumer(EndOfSuperstepEvent.INSTANCE); subpartition.add(eventBuffer); assertEquals(1, availablityListener.getNumNotifications()); assertEquals(0, availablityListener.getNumPriorityEvents()); subpartition.add(createFilledFinishedBufferConsumer(4)); assertEquals(1, availablityListener.getNumNotifications()); assertEquals(0, availablityListener.getNumPriorityEvents()); CheckpointOptions options = new CheckpointOptions( CheckpointType.CHECKPOINT, new CheckpointStorageLocationReference(new byte[]{0, 1, 2}), true, true); BufferConsumer barrierBuffer = EventSerializer.toBufferConsumer(new CheckpointBarrier(0, 0, options)); subpartition.add(barrierBuffer, true); assertEquals(2, availablityListener.getNumNotifications()); assertEquals(0, availablityListener.getNumPriorityEvents()); List<Buffer> inflight = subpartition.requestInflightBufferSnapshot(); assertEquals(Arrays.asList(1, 2, 4), inflight.stream().map(Buffer::getSize).collect(Collectors.toList())); inflight.forEach(Buffer::recycleBuffer); assertNextEvent(readView, barrierBuffer.getWrittenBytes(), CheckpointBarrier.class, true, 2, false, true); assertNextBuffer(readView, 1, true, 1, false, true); assertNextBuffer(readView, 2, true, 0, true, true); assertNextEvent(readView, eventBuffer.getWrittenBytes(), EndOfSuperstepEvent.class, false, 0, false, true); assertNextBuffer(readView, 4, false, 0, false, true); assertNoNextBuffer(readView); }