org.apache.flink.runtime.state.CheckpointStorageLocationReference Java Examples

The following examples show how to use org.apache.flink.runtime.state.CheckpointStorageLocationReference. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SubtaskCheckpointCoordinatorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private boolean initCheckpoint(boolean unalignedCheckpointEnabled, CheckpointType checkpointType) throws IOException {
	class MockWriter extends ChannelStateWriterImpl.NoOpChannelStateWriter {
		private boolean started;
		@Override
		public void start(long checkpointId, CheckpointOptions checkpointOptions) {
			started = true;
		}
	}

	MockWriter writer = new MockWriter();
	SubtaskCheckpointCoordinator coordinator = coordinator(unalignedCheckpointEnabled, writer);
	CheckpointStorageLocationReference locationReference = CheckpointStorageLocationReference.getDefault();
	CheckpointOptions options = new CheckpointOptions(checkpointType, locationReference, true, unalignedCheckpointEnabled);
	coordinator.initCheckpoint(1L, options);
	return writer.started;
}
 
Example #2
Source File: FsCheckpointStorage.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public CheckpointStorageLocation initializeLocationForCheckpoint(long checkpointId) throws IOException {
	checkArgument(checkpointId >= 0, "Illegal negative checkpoint id: %d.", checkpointId);
	checkArgument(baseLocationsInitialized, "The base checkpoint location has not been initialized.");

	// prepare all the paths needed for the checkpoints
	final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId);

	// create the checkpoint exclusive directory
	fileSystem.mkdirs(checkpointDir);

	return new FsCheckpointStorageLocation(
			fileSystem,
			checkpointDir,
			sharedStateDirectory,
			taskOwnedStateDirectory,
			CheckpointStorageLocationReference.getDefault(),
			fileSizeThreshold,
			writeBufferSize);
}
 
Example #3
Source File: FsCheckpointStorageLocation.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public FsCheckpointStorageLocation(
		FileSystem fileSystem,
		Path checkpointDir,
		Path sharedStateDir,
		Path taskOwnedStateDir,
		CheckpointStorageLocationReference reference,
		int fileStateSizeThreshold) {

	super(fileSystem, checkpointDir, sharedStateDir, fileStateSizeThreshold);

	checkArgument(fileStateSizeThreshold >= 0);

	this.fileSystem = checkNotNull(fileSystem);
	this.checkpointDirectory = checkNotNull(checkpointDir);
	this.sharedStateDirectory = checkNotNull(sharedStateDir);
	this.taskOwnedStateDirectory = checkNotNull(taskOwnedStateDir);
	this.reference = checkNotNull(reference);

	// the metadata file should not have entropy in its path
	Path metadataDir = EntropyInjector.removeEntropyMarkerIfPresent(fileSystem, checkpointDir);

	this.metadataFilePath = new Path(metadataDir, AbstractFsCheckpointStorage.METADATA_FILE_NAME);
	this.fileStateSizeThreshold = fileStateSizeThreshold;
}
 
Example #4
Source File: FsCheckpointStorage.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public CheckpointStorageLocation initializeLocationForCheckpoint(long checkpointId) throws IOException {
	checkArgument(checkpointId >= 0);

	// prepare all the paths needed for the checkpoints
	final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId);

	// create the checkpoint exclusive directory
	fileSystem.mkdirs(checkpointDir);

	return new FsCheckpointStorageLocation(
			fileSystem,
			checkpointDir,
			sharedStateDirectory,
			taskOwnedStateDirectory,
			CheckpointStorageLocationReference.getDefault(),
			fileSizeThreshold);
}
 
Example #5
Source File: FsCheckpointStorageTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * This test checks that the expected mkdirs action for checkpoint storage,
 * only be called when initializeBaseLocations and not called when resolveCheckpointStorageLocation.
 */
@Test
public void testStorageLocationMkdirs() throws Exception {
	FsCheckpointStorage storage = new FsCheckpointStorage(
			randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE);

	File baseDir = new File(storage.getCheckpointsDirectory().getPath());
	assertFalse(baseDir.exists());

	// mkdirs would only be called when initializeBaseLocations
	storage.initializeBaseLocations();
	assertTrue(baseDir.exists());

	// mkdir would not be called when resolveCheckpointStorageLocation
	storage = new FsCheckpointStorage(
			randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE);

	FsCheckpointStorageLocation location = (FsCheckpointStorageLocation)
			storage.resolveCheckpointStorageLocation(177, CheckpointStorageLocationReference.getDefault());

	Path checkpointPath = location.getCheckpointDirectory();
	File checkpointDir = new File(checkpointPath.getPath());
	assertFalse(checkpointDir.exists());
}
 
Example #6
Source File: PendingCheckpointTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private PendingCheckpoint createPendingCheckpoint(CheckpointProperties props, Executor executor) throws IOException {

		final Path checkpointDir = new Path(tmpFolder.newFolder().toURI());
		final FsCheckpointStorageLocation location = new FsCheckpointStorageLocation(
				LocalFileSystem.getSharedInstance(),
				checkpointDir, checkpointDir, checkpointDir,
				CheckpointStorageLocationReference.getDefault(),
				1024);

		final Map<ExecutionAttemptID, ExecutionVertex> ackTasks = new HashMap<>(ACK_TASKS);

		return new PendingCheckpoint(
			new JobID(),
			0,
			1,
			ackTasks,
			props,
			location,
			executor);
	}
 
Example #7
Source File: AlternatingCheckpointBarrierHandlerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testPreviousHandlerReset() throws Exception {
	SingleInputGate inputGate = new SingleInputGateBuilder().setNumberOfChannels(2).build();
	inputGate.setInputChannels(new TestInputChannel(inputGate, 0), new TestInputChannel(inputGate, 1));
	TestInvokable target = new TestInvokable();
	CheckpointBarrierAligner alignedHandler = new CheckpointBarrierAligner("test", target, inputGate);
	CheckpointBarrierUnaligner unalignedHandler = new CheckpointBarrierUnaligner(TestSubtaskCheckpointCoordinator.INSTANCE, "test", target, inputGate);
	AlternatingCheckpointBarrierHandler barrierHandler = new AlternatingCheckpointBarrierHandler(alignedHandler, unalignedHandler, target);

	for (int i = 0; i < 4; i++) {
		int channel = i % 2;
		CheckpointType type = channel == 0 ? CHECKPOINT : SAVEPOINT;
		barrierHandler.processBarrier(new CheckpointBarrier(i, 0, new CheckpointOptions(type, CheckpointStorageLocationReference.getDefault())), new InputChannelInfo(0, channel));
		assertEquals(type.isSavepoint(), alignedHandler.isCheckpointPending());
		assertNotEquals(alignedHandler.isCheckpointPending(), unalignedHandler.isCheckpointPending());

		if (!type.isSavepoint()) {
			assertFalse(barrierHandler.getAllBarriersReceivedFuture(i).isDone());
			assertInflightDataEquals(unalignedHandler, barrierHandler, i, inputGate.getNumberOfInputChannels());
		}
	}
}
 
Example #8
Source File: FsCheckpointStorageTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * This test checks that no mkdirs is called by the checkpoint storage location when resolved.
 */
@Test
public void testStorageLocationDoesNotMkdirs() throws Exception {
	FsCheckpointStorage storage = new FsCheckpointStorage(
			randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD);

	File baseDir =  new File(storage.getCheckpointsDirectory().getPath());
	assertTrue(baseDir.exists());

	FsCheckpointStorageLocation location = (FsCheckpointStorageLocation)
			storage.resolveCheckpointStorageLocation(177, CheckpointStorageLocationReference.getDefault());

	Path checkpointPath = location.getCheckpointDirectory();
	File checkpointDir = new File(checkpointPath.getPath());
	assertFalse(checkpointDir.exists());
}
 
Example #9
Source File: FsStorageLocationReferenceTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testEncodeAndDecode() throws Exception {
	final Path path = randomPath(new Random());

	try {
		CheckpointStorageLocationReference ref = encodePathAsReference(path);
		Path decoded = decodePathFromReference(ref);

		assertEquals(path, decoded);
	}
	catch (Exception | Error e) {
		// if something goes wrong, help by printing the problematic path
		log.error("ERROR FOR PATH " + path);
		throw e;
	}
}
 
Example #10
Source File: FsCheckpointStorageTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testResolveCheckpointStorageLocation() throws Exception {
	final FileSystem checkpointFileSystem = mock(FileSystem.class);
	final FsCheckpointStorage storage = new FsCheckpointStorage(
		new TestingPath("hdfs:///checkpoint/", checkpointFileSystem),
		null,
		new JobID(),
		FILE_SIZE_THRESHOLD,
		WRITE_BUFFER_SIZE);

	final FsCheckpointStorageLocation checkpointStreamFactory =
		(FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(1L, CheckpointStorageLocationReference.getDefault());
	assertEquals(checkpointFileSystem, checkpointStreamFactory.getFileSystem());

	final CheckpointStorageLocationReference savepointLocationReference =
		AbstractFsCheckpointStorage.encodePathAsReference(new Path("file:///savepoint/"));

	final FsCheckpointStorageLocation savepointStreamFactory =
		(FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(2L, savepointLocationReference);
	final FileSystem fileSystem = savepointStreamFactory.getFileSystem();
	assertTrue(fileSystem instanceof LocalFileSystem);
}
 
Example #11
Source File: FsCheckpointStorageTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testResolveCheckpointStorageLocation() throws Exception {
	final FileSystem checkpointFileSystem = mock(FileSystem.class);
	final FsCheckpointStorage storage = new FsCheckpointStorage(
		new TestingPath("hdfs:///checkpoint/", checkpointFileSystem),
		null,
		new JobID(),
		FILE_SIZE_THRESHOLD);

	final FsCheckpointStorageLocation checkpointStreamFactory =
		(FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(1L, CheckpointStorageLocationReference.getDefault());
	assertEquals(checkpointFileSystem, checkpointStreamFactory.getFileSystem());

	final CheckpointStorageLocationReference savepointLocationReference =
		AbstractFsCheckpointStorage.encodePathAsReference(new Path("file:///savepoint/"));

	final FsCheckpointStorageLocation savepointStreamFactory =
		(FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(2L, savepointLocationReference);
	final FileSystem fileSystem = savepointStreamFactory.getFileSystem();
	assertTrue(fileSystem instanceof LocalFileSystem);
}
 
Example #12
Source File: PendingCheckpointTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private PendingCheckpoint createPendingCheckpoint(CheckpointProperties props, Executor executor) throws IOException {

		final Path checkpointDir = new Path(tmpFolder.newFolder().toURI());
		final FsCheckpointStorageLocation location = new FsCheckpointStorageLocation(
				LocalFileSystem.getSharedInstance(),
				checkpointDir, checkpointDir, checkpointDir,
				CheckpointStorageLocationReference.getDefault(),
				1024,
				4096);

		final Map<ExecutionAttemptID, ExecutionVertex> ackTasks = new HashMap<>(ACK_TASKS);

		return new PendingCheckpoint(
			new JobID(),
			0,
			1,
			ackTasks,
			props,
			location,
			executor);
	}
 
Example #13
Source File: FsCheckpointStorage.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public CheckpointStorageLocation initializeLocationForCheckpoint(long checkpointId) throws IOException {
	checkArgument(checkpointId >= 0);

	// prepare all the paths needed for the checkpoints
	final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId);

	// create the checkpoint exclusive directory
	fileSystem.mkdirs(checkpointDir);

	return new FsCheckpointStorageLocation(
			fileSystem,
			checkpointDir,
			sharedStateDirectory,
			taskOwnedStateDirectory,
			CheckpointStorageLocationReference.getDefault(),
			fileSizeThreshold,
			writeBufferSize);
}
 
Example #14
Source File: FsCheckpointStorageTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * This test checks that no mkdirs is called by the checkpoint storage location when resolved.
 */
@Test
public void testStorageLocationDoesNotMkdirs() throws Exception {
	FsCheckpointStorage storage = new FsCheckpointStorage(
			randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE);

	File baseDir =  new File(storage.getCheckpointsDirectory().getPath());
	assertTrue(baseDir.exists());

	FsCheckpointStorageLocation location = (FsCheckpointStorageLocation)
			storage.resolveCheckpointStorageLocation(177, CheckpointStorageLocationReference.getDefault());

	Path checkpointPath = location.getCheckpointDirectory();
	File checkpointDir = new File(checkpointPath.getPath());
	assertFalse(checkpointDir.exists());
}
 
Example #15
Source File: FsCheckpointStorageTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testResolveCheckpointStorageLocation() throws Exception {
	final FileSystem checkpointFileSystem = mock(FileSystem.class);
	final FsCheckpointStorage storage = new FsCheckpointStorage(
		new TestingPath("hdfs:///checkpoint/", checkpointFileSystem),
		null,
		new JobID(),
		FILE_SIZE_THRESHOLD,
		WRITE_BUFFER_SIZE);

	final FsCheckpointStorageLocation checkpointStreamFactory =
		(FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(1L, CheckpointStorageLocationReference.getDefault());
	assertEquals(checkpointFileSystem, checkpointStreamFactory.getFileSystem());

	final CheckpointStorageLocationReference savepointLocationReference =
		AbstractFsCheckpointStorage.encodePathAsReference(new Path("file:///savepoint/"));

	final FsCheckpointStorageLocation savepointStreamFactory =
		(FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(2L, savepointLocationReference);
	final FileSystem fileSystem = savepointStreamFactory.getFileSystem();
	assertTrue(fileSystem instanceof LocalFileSystem);
}
 
Example #16
Source File: FsStorageLocationReferenceTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testEncodeAndDecode() throws Exception {
	final Path path = randomPath(new Random());

	try {
		CheckpointStorageLocationReference ref = encodePathAsReference(path);
		Path decoded = decodePathFromReference(ref);

		assertEquals(path, decoded);
	}
	catch (Exception | Error e) {
		// if something goes wrong, help by printing the problematic path
		log.error("ERROR FOR PATH " + path);
		throw e;
	}
}
 
Example #17
Source File: AlternatingCheckpointBarrierHandlerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testOutOfOrderBarrier() throws Exception {
	SingleInputGate inputGate = new SingleInputGateBuilder().setNumberOfChannels(2).build();
	inputGate.setInputChannels(new TestInputChannel(inputGate, 0), new TestInputChannel(inputGate, 1));
	TestInvokable target = new TestInvokable();
	CheckpointBarrierAligner alignedHandler = new CheckpointBarrierAligner("test", target, inputGate);
	CheckpointBarrierUnaligner unalignedHandler = new CheckpointBarrierUnaligner(TestSubtaskCheckpointCoordinator.INSTANCE, "test", target, inputGate);
	AlternatingCheckpointBarrierHandler barrierHandler = new AlternatingCheckpointBarrierHandler(alignedHandler, unalignedHandler, target);

	long checkpointId = 10;
	long outOfOrderSavepointId = 5;
	long initialAlignedCheckpointId = alignedHandler.getLatestCheckpointId();

	barrierHandler.processBarrier(new CheckpointBarrier(checkpointId, 0, new CheckpointOptions(CHECKPOINT, CheckpointStorageLocationReference.getDefault())), new InputChannelInfo(0, 0));
	barrierHandler.processBarrier(new CheckpointBarrier(outOfOrderSavepointId, 0, new CheckpointOptions(SAVEPOINT, CheckpointStorageLocationReference.getDefault())), new InputChannelInfo(0, 1));

	assertEquals(checkpointId, barrierHandler.getLatestCheckpointId());
	assertInflightDataEquals(unalignedHandler, barrierHandler, checkpointId, inputGate.getNumberOfInputChannels());
	assertEquals(initialAlignedCheckpointId, alignedHandler.getLatestCheckpointId());
}
 
Example #18
Source File: SubtaskCheckpointCoordinatorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testSkipChannelStateForSavepoints() throws Exception {
	SubtaskCheckpointCoordinator coordinator = new MockSubtaskCheckpointCoordinatorBuilder()
		.setUnalignedCheckpointEnabled(true)
		.setPrepareInputSnapshot((u1, u2) -> {
			fail("should not prepare input snapshot for savepoint");
			return null;
		}).build();

	coordinator.checkpointState(
		new CheckpointMetaData(0, 0),
		new CheckpointOptions(SAVEPOINT, CheckpointStorageLocationReference.getDefault()),
		new CheckpointMetrics(),
		new OperatorChain<>(new NoOpStreamTask<>(new DummyEnvironment()), new NonRecordWriter<>()),
		() -> false);
}
 
Example #19
Source File: CheckpointOptionsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSavepointNeedsAlignment() {
	CheckpointStorageLocationReference location = CheckpointStorageLocationReference.getDefault();
	assertTrue(new CheckpointOptions(SAVEPOINT, location, true, true).needsAlignment());
	assertFalse(new CheckpointOptions(SAVEPOINT, location, false, true).needsAlignment());
	assertTrue(new CheckpointOptions(SAVEPOINT, location, true, false).needsAlignment());
	assertFalse(new CheckpointOptions(SAVEPOINT, location, false, false).needsAlignment());
}
 
Example #20
Source File: RemoteInputChannelTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testNoNotifyOnSavepoint() throws IOException {
	TestBufferReceivedListener listener = new TestBufferReceivedListener();
	SingleInputGate inputGate = new SingleInputGateBuilder().build();
	inputGate.registerBufferReceivedListener(listener);
	RemoteInputChannel channel = InputChannelBuilder.newBuilder().buildRemoteChannel(inputGate);
	channel.onBuffer(toBuffer(new CheckpointBarrier(123L, 123L, new CheckpointOptions(SAVEPOINT, CheckpointStorageLocationReference.getDefault()))), 0, 0);
	channel.checkError();
	assertTrue(listener.notifiedOnBarriers.isEmpty());
}
 
Example #21
Source File: CheckpointOptionsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSavepoint() throws Exception {
	final Random rnd = new Random();
	final byte[] locationBytes = new byte[rnd.nextInt(41) + 1];
	rnd.nextBytes(locationBytes);

	final CheckpointOptions options = new CheckpointOptions(
			CheckpointType.values()[rnd.nextInt(CheckpointType.values().length)],
			new CheckpointStorageLocationReference(locationBytes));

	final CheckpointOptions copy = CommonTestUtils.createCopySerializable(options);
	assertEquals(options.getCheckpointType(), copy.getCheckpointType());
	assertArrayEquals(locationBytes, copy.getTargetLocation().getReferenceBytes());
}
 
Example #22
Source File: MemoryBackendCheckpointStorage.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public CheckpointStreamFactory resolveCheckpointStorageLocation(
		long checkpointId,
		CheckpointStorageLocationReference reference) {

	// no matter where the checkpoint goes, we always return the storage location that stores
	// state inline with the state handles.
	return new MemCheckpointStreamFactory(maxStateSize);
}
 
Example #23
Source File: LocalInputChannelTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testNoNotifyOnSavepoint() throws IOException {
	TestBufferReceivedListener listener = new TestBufferReceivedListener();
	SingleInputGate inputGate = new SingleInputGateBuilder().build();
	inputGate.registerBufferReceivedListener(listener);
	LocalInputChannel channel = InputChannelBuilder.newBuilder().buildLocalChannel(inputGate);
	CheckpointBarrier barrier = new CheckpointBarrier(123L, 123L, new CheckpointOptions(SAVEPOINT, CheckpointStorageLocationReference.getDefault()));
	channel.notifyPriorityEvent(new BufferConsumer(toBuffer(barrier).getMemorySegment(), FreeingBufferRecycler.INSTANCE, getDataType(barrier)));
	channel.checkError();
	assertTrue(listener.notifiedOnBarriers.isEmpty());
}
 
Example #24
Source File: FsCheckpointStorage.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public CheckpointStreamFactory resolveCheckpointStorageLocation(
		long checkpointId,
		CheckpointStorageLocationReference reference) throws IOException {

	if (reference.isDefaultReference()) {
		// default reference, construct the default location for that particular checkpoint
		final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId);

		return new FsCheckpointStorageLocation(
				fileSystem,
				checkpointDir,
				sharedStateDirectory,
				taskOwnedStateDirectory,
				reference,
				fileSizeThreshold,
				writeBufferSize);
	}
	else {
		// location encoded in the reference
		final Path path = decodePathFromReference(reference);

		return new FsCheckpointStorageLocation(
				path.getFileSystem(),
				path,
				path,
				path,
				reference,
				fileSizeThreshold,
				writeBufferSize);
	}
}
 
Example #25
Source File: SylphFsCheckpointStorage.java    From sylph with Apache License 2.0 5 votes vote down vote up
@Override
public CheckpointStreamFactory resolveCheckpointStorageLocation(
        long checkpointId,
        CheckpointStorageLocationReference reference)
        throws IOException
{
    if (reference.isDefaultReference()) {
        // default reference, construct the default location for that particular checkpoint
        final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId);

        return new FsCheckpointStorageLocation(
                fileSystem,
                checkpointDir,
                sharedStateDirectory,
                taskOwnedStateDirectory,
                reference,
                fileSizeThreshold,
                writeBufferSize);
    }
    else {
        // location encoded in the reference
        final Path path = decodePathFromReference(reference);

        return new FsCheckpointStorageLocation(
                path.getFileSystem(),
                path,
                path,
                path,
                reference,
                fileSizeThreshold,
                writeBufferSize);
    }
}
 
Example #26
Source File: SavepointOutputFormat.java    From flink with Apache License 2.0 5 votes vote down vote up
private static CheckpointStorageLocation createSavepointLocation(Path location) throws IOException {
	final CheckpointStorageLocationReference reference = AbstractFsCheckpointStorage.encodePathAsReference(location);
	return new FsCheckpointStorageLocation(
		location.getFileSystem(),
		location,
		location,
		location,
		reference,
		(int) CheckpointingOptions.FS_SMALL_FILE_THRESHOLD.defaultValue().getBytes(),
		CheckpointingOptions.FS_WRITE_BUFFER_SIZE.defaultValue());
}
 
Example #27
Source File: CheckpointOptionsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testCheckpointNeedsAlignment() {
	CheckpointStorageLocationReference location = CheckpointStorageLocationReference.getDefault();
	assertFalse(new CheckpointOptions(CHECKPOINT, location, true, true).needsAlignment());
	assertTrue(new CheckpointOptions(CHECKPOINT, location, true, false).needsAlignment());
	assertFalse(new CheckpointOptions(CHECKPOINT, location, false, true).needsAlignment());
	assertFalse(new CheckpointOptions(CHECKPOINT, location, false, false).needsAlignment());
}
 
Example #28
Source File: AbstractFsCheckpointStorage.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Encodes the given path as a reference in bytes. The path is encoded as a UTF-8 string
 * and prepended as a magic number.
 *
 * @param path The path to encode.
 * @return The location reference.
 */
public static CheckpointStorageLocationReference encodePathAsReference(Path path) {
	byte[] refBytes = path.toString().getBytes(StandardCharsets.UTF_8);
	byte[] bytes = new byte[REFERENCE_MAGIC_NUMBER.length + refBytes.length];

	System.arraycopy(REFERENCE_MAGIC_NUMBER, 0, bytes, 0, REFERENCE_MAGIC_NUMBER.length);
	System.arraycopy(refBytes, 0, bytes, REFERENCE_MAGIC_NUMBER.length, refBytes.length);

	return new CheckpointStorageLocationReference(bytes);
}
 
Example #29
Source File: SylphFsCheckpointStorage.java    From sylph with Apache License 2.0 5 votes vote down vote up
@Override
protected CheckpointStorageLocation createSavepointLocation(FileSystem fs, Path location)
        throws IOException
{
    final CheckpointStorageLocationReference reference = encodePathAsReference(location);
    return new FsCheckpointStorageLocation(fs, location, location, location, reference, fileSizeThreshold, writeBufferSize);
}
 
Example #30
Source File: PipelinedSubpartitionWithReadViewTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testBarrierOvertaking() throws Exception {
	subpartition.add(createFilledFinishedBufferConsumer(1));
	assertEquals(0, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	subpartition.add(createFilledFinishedBufferConsumer(2));
	assertEquals(1, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	BufferConsumer eventBuffer = EventSerializer.toBufferConsumer(EndOfSuperstepEvent.INSTANCE);
	subpartition.add(eventBuffer);
	assertEquals(1, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	subpartition.add(createFilledFinishedBufferConsumer(4));
	assertEquals(1, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	CheckpointOptions options = new CheckpointOptions(
		CheckpointType.CHECKPOINT,
		new CheckpointStorageLocationReference(new byte[]{0, 1, 2}),
		true,
		true);
	BufferConsumer barrierBuffer = EventSerializer.toBufferConsumer(new CheckpointBarrier(0, 0, options));
	subpartition.add(barrierBuffer, true);
	assertEquals(2, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	List<Buffer> inflight = subpartition.requestInflightBufferSnapshot();
	assertEquals(Arrays.asList(1, 2, 4), inflight.stream().map(Buffer::getSize).collect(Collectors.toList()));
	inflight.forEach(Buffer::recycleBuffer);

	assertNextEvent(readView, barrierBuffer.getWrittenBytes(), CheckpointBarrier.class, true, 2, false, true);
	assertNextBuffer(readView, 1, true, 1, false, true);
	assertNextBuffer(readView, 2, true, 0, true, true);
	assertNextEvent(readView, eventBuffer.getWrittenBytes(), EndOfSuperstepEvent.class, false, 0, false, true);
	assertNextBuffer(readView, 4, false, 0, false, true);
	assertNoNextBuffer(readView);
}