Java Code Examples for org.apache.flink.runtime.checkpoint.CheckpointOptions#forCheckpointWithDefaultLocation()

@Test
public void testSerializeDeserializeEvent() throws Exception {
	AbstractEvent[] events = {
			EndOfPartitionEvent.INSTANCE,
			EndOfSuperstepEvent.INSTANCE,
			new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forCheckpointWithDefaultLocation()),
			new TestTaskEvent(Math.random(), 12361231273L),
			new CancelCheckpointMarker(287087987329842L)
	};

	for (AbstractEvent evt : events) {
		ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
		assertTrue(serializedEvent.hasRemaining());

		AbstractEvent deserialized =
				EventSerializer.fromSerializedEvent(serializedEvent, getClass().getClassLoader());
		assertNotNull(deserialized);
		assertEquals(evt, deserialized);
	}
}
 

@Test
public void testSerializeDeserializeEvent() throws Exception {
	AbstractEvent[] events = {
			EndOfPartitionEvent.INSTANCE,
			EndOfSuperstepEvent.INSTANCE,
			new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forCheckpointWithDefaultLocation()),
			new TestTaskEvent(Math.random(), 12361231273L),
			new CancelCheckpointMarker(287087987329842L)
	};

	for (AbstractEvent evt : events) {
		ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
		assertTrue(serializedEvent.hasRemaining());

		AbstractEvent deserialized =
				EventSerializer.fromSerializedEvent(serializedEvent, getClass().getClassLoader());
		assertNotNull(deserialized);
		assertEquals(evt, deserialized);
	}
}
 

void testNonConcurrentSnapshotTransformerAccess() throws Exception {
	List<TestState> testStates = Arrays.asList(
		new TestValueState(),
		new TestListState(),
		new TestMapState()
	);

	for (TestState state : testStates) {
		for (int i = 0; i < 100; i++) {
			backend.setCurrentKey(i);
			state.setToRandomValue();
		}

		CheckpointOptions checkpointOptions = CheckpointOptions.forCheckpointWithDefaultLocation();

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot1 =
			backend.snapshot(1L, 0L, streamFactory, checkpointOptions);

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot2 =
			backend.snapshot(2L, 0L, streamFactory, checkpointOptions);

		Thread runner1 = new Thread(snapshot1, "snapshot1");
		runner1.start();
		Thread runner2 = new Thread(snapshot2, "snapshot2");
		runner2.start();

		runner1.join();
		runner2.join();

		snapshot1.get();
		snapshot2.get();
	}
}
 

protected StateBackendTestContext(TtlTimeProvider timeProvider) {
	this.timeProvider = Preconditions.checkNotNull(timeProvider);
	this.stateBackend = Preconditions.checkNotNull(createStateBackend());
	this.checkpointOptions = CheckpointOptions.forCheckpointWithDefaultLocation();
	this.checkpointStreamFactory = createCheckpointStreamFactory();
	this.sharedStateRegistry = new SharedStateRegistry();
	this.snapshots = new ArrayList<>();
}
 

/**
 * InputGate on CheckpointBarrier can enqueue a mailbox action to execute and StreamTaskNetworkInput must
 * allow this action to execute before processing a following record.
 */
@Test
public void testNoDataProcessedAfterCheckpointBarrier() throws Exception {
	CheckpointBarrier barrier = new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation());

	List<BufferOrEvent> buffers = new ArrayList<>(2);
	buffers.add(new BufferOrEvent(barrier, new InputChannelInfo(0, 0)));
	buffers.add(createDataBuffer());

	VerifyRecordsDataOutput output = new VerifyRecordsDataOutput<>();
	StreamTaskNetworkInput input = createStreamTaskNetworkInput(buffers, output);

	assertHasNextElement(input, output);
	assertEquals(0, output.getNumberOfEmittedRecords());
}
 

protected StateBackendTestContext(TtlTimeProvider timeProvider) {
	this.timeProvider = Preconditions.checkNotNull(timeProvider);
	this.stateBackend = Preconditions.checkNotNull(createStateBackend());
	this.checkpointOptions = CheckpointOptions.forCheckpointWithDefaultLocation();
	this.checkpointStreamFactory = createCheckpointStreamFactory();
	this.sharedStateRegistry = new SharedStateRegistry();
	this.snapshots = new ArrayList<>();
}
 

void testNonConcurrentSnapshotTransformerAccess() throws Exception {
	List<TestState> testStates = Arrays.asList(
		new TestValueState(),
		new TestListState(),
		new TestMapState()
	);

	for (TestState state : testStates) {
		for (int i = 0; i < 100; i++) {
			backend.setCurrentKey(i);
			state.setToRandomValue();
		}

		CheckpointOptions checkpointOptions = CheckpointOptions.forCheckpointWithDefaultLocation();

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot1 =
			backend.snapshot(1L, 0L, streamFactory, checkpointOptions);

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot2 =
			backend.snapshot(2L, 0L, streamFactory, checkpointOptions);

		Thread runner1 = new Thread(snapshot1, "snapshot1");
		runner1.start();
		Thread runner2 = new Thread(snapshot2, "snapshot2");
		runner2.start();

		runner1.join();
		runner2.join();

		snapshot1.get();
		snapshot2.get();
	}
}
 

@Test
public void testTriggerAndConfirmCheckpoint() {
	try {
		NotifyCheckpointComplete cc = new NotifyCheckpointComplete(new JobID(), new ExecutionAttemptID(), 45287698767345L, 467L);
		testSerializabilityEqualsHashCode(cc);

		TriggerCheckpoint tc = new TriggerCheckpoint(new JobID(), new ExecutionAttemptID(), 347652734L, 7576752L, CheckpointOptions.forCheckpointWithDefaultLocation());
		testSerializabilityEqualsHashCode(tc);

	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

/**
 * Tests broadcasting events when no records have been emitted yet.
 */
@Test
public void testBroadcastEventNoRecords() throws Exception {
	int numberOfChannels = 4;
	int bufferSize = 32;

	@SuppressWarnings("unchecked")
	Queue<BufferConsumer>[] queues = new Queue[numberOfChannels];
	for (int i = 0; i < numberOfChannels; i++) {
		queues[i] = new ArrayDeque<>();
	}

	TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize);

	ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider);
	RecordWriter<ByteArrayIO> writer = new RecordWriter<>(partitionWriter);
	CheckpointBarrier barrier = new CheckpointBarrier(Integer.MAX_VALUE + 919192L, Integer.MAX_VALUE + 18828228L, CheckpointOptions.forCheckpointWithDefaultLocation());

	// No records emitted yet, broadcast should not request a buffer
	writer.broadcastEvent(barrier);

	assertEquals(0, bufferProvider.getNumberOfCreatedBuffers());

	for (int i = 0; i < numberOfChannels; i++) {
		assertEquals(1, queues[i].size());
		BufferOrEvent boe = parseBuffer(queues[i].remove(), i);
		assertTrue(boe.isEvent());
		assertEquals(barrier, boe.getEvent());
		assertEquals(0, queues[i].size());
	}
}
 

/**
 * Tests broadcasting events when no records have been emitted yet.
 */
@Test
public void testBroadcastEventNoRecords() throws Exception {
	int numberOfChannels = 4;
	int bufferSize = 32;

	@SuppressWarnings("unchecked")
	Queue<BufferConsumer>[] queues = new Queue[numberOfChannels];
	for (int i = 0; i < numberOfChannels; i++) {
		queues[i] = new ArrayDeque<>();
	}

	TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize);

	ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider);
	RecordWriter<ByteArrayIO> writer = createRecordWriter(partitionWriter);
	CheckpointBarrier barrier = new CheckpointBarrier(Integer.MAX_VALUE + 919192L, Integer.MAX_VALUE + 18828228L, CheckpointOptions.forCheckpointWithDefaultLocation());

	// No records emitted yet, broadcast should not request a buffer
	writer.broadcastEvent(barrier);

	assertEquals(0, bufferProvider.getNumberOfCreatedBuffers());

	for (int i = 0; i < numberOfChannels; i++) {
		assertEquals(1, queues[i].size());
		BufferOrEvent boe = parseBuffer(queues[i].remove(), i);
		assertTrue(boe.isEvent());
		assertEquals(barrier, boe.getEvent());
		assertEquals(0, queues[i].size());
	}
}
 

@Test
public void testCheckpointBarrierSerialization() throws Exception {
	long id = Integer.MAX_VALUE + 123123L;
	long timestamp = Integer.MAX_VALUE + 1228L;

	CheckpointOptions checkpoint = CheckpointOptions.forCheckpointWithDefaultLocation();
	testCheckpointBarrierSerialization(id, timestamp, checkpoint);

	final byte[] reference = new byte[] { 15, 52, 52, 11, 0, 0, 0, 0, -1, -23, -19, 35 };

	CheckpointOptions savepoint = new CheckpointOptions(
			CheckpointType.SAVEPOINT, new CheckpointStorageLocationReference(reference));
	testCheckpointBarrierSerialization(id, timestamp, savepoint);
}
 

@Test
public void testTriggerAndConfirmCheckpoint() {
	try {
		NotifyCheckpointComplete cc = new NotifyCheckpointComplete(new JobID(), new ExecutionAttemptID(), 45287698767345L, 467L);
		testSerializabilityEqualsHashCode(cc);

		TriggerCheckpoint tc = new TriggerCheckpoint(new JobID(), new ExecutionAttemptID(), 347652734L, 7576752L, CheckpointOptions.forCheckpointWithDefaultLocation());
		testSerializabilityEqualsHashCode(tc);

	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

private TaskStateSnapshot executeAndWaitForCheckpoint(
		long checkpointId,
		TaskStateSnapshot initialSnapshot,
		IntStream expectedRecords) throws Exception {

	try (StreamTaskMailboxTestHarness<Integer> testHarness = createTestHarness(checkpointId, initialSnapshot)) {
		// Add records to the splits.
		MockSourceSplit split = getAndMaybeAssignSplit(testHarness);
		// Add records to the split and update expected output.
		addRecords(split, NUM_RECORDS);
		// Process all the records.
		processUntil(testHarness, () -> !testHarness.getStreamTask().inputProcessor.getAvailableFuture().isDone());

		// Trigger a checkpoint.
		CheckpointOptions checkpointOptions = CheckpointOptions.forCheckpointWithDefaultLocation();
		OneShotLatch waitForAcknowledgeLatch = new OneShotLatch();
		testHarness.taskStateManager.setWaitForReportLatch(waitForAcknowledgeLatch);
		CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, checkpointId);
		Future<Boolean> checkpointFuture =
				testHarness
						.getStreamTask()
						.triggerCheckpointAsync(checkpointMetaData, checkpointOptions, false);

		// Wait until the checkpoint finishes.
		// We have to mark the source reader as available here, otherwise the runMailboxStep() call after
		// checkpiont is completed will block.
		getSourceReaderFromTask(testHarness).markAvailable();
		processUntil(testHarness, checkpointFuture::isDone);
		waitForAcknowledgeLatch.await();

		// Build expected output to verify the results
		Queue<Object> expectedOutput = new LinkedList<>();
		expectedRecords.forEach(r -> expectedOutput.offer(new StreamRecord<>(r, TimestampAssigner.NO_TIMESTAMP)));
		// Add barrier to the expected output.
		expectedOutput.add(new CheckpointBarrier(checkpointId, checkpointId, checkpointOptions));

		assertEquals(checkpointId, testHarness.taskStateManager.getReportedCheckpointId());
		assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

		return testHarness.taskStateManager.getLastJobManagerTaskStateSnapshot();
	}
}
 

private static BufferOrEvent createBarrier(long id, int channel) {
	return new BufferOrEvent(new CheckpointBarrier(id, System.currentTimeMillis(), CheckpointOptions.forCheckpointWithDefaultLocation()), channel);
}
 

private static BufferOrEvent createBarrier(long id, int channel) {
	return new BufferOrEvent(new CheckpointBarrier(id, System.currentTimeMillis(), CheckpointOptions.forCheckpointWithDefaultLocation()), channel);
}
 

/**
 * Tests broadcasting events when records have been emitted. The emitted
 * records cover all three {@link SerializationResult} types.
 */
@Test
public void testBroadcastEventMixedRecords() throws Exception {
	Random rand = new XORShiftRandom();
	int numberOfChannels = 4;
	int bufferSize = 32;
	int lenBytes = 4; // serialized length

	@SuppressWarnings("unchecked")
	Queue<BufferConsumer>[] queues = new Queue[numberOfChannels];
	for (int i = 0; i < numberOfChannels; i++) {
		queues[i] = new ArrayDeque<>();
	}

	TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize);

	ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider);
	RecordWriter<ByteArrayIO> writer = new RecordWriter<>(partitionWriter);
	CheckpointBarrier barrier = new CheckpointBarrier(Integer.MAX_VALUE + 1292L, Integer.MAX_VALUE + 199L, CheckpointOptions.forCheckpointWithDefaultLocation());

	// Emit records on some channels first (requesting buffers), then
	// broadcast the event. The record buffers should be emitted first, then
	// the event. After the event, no new buffer should be requested.

	// (i) Smaller than the buffer size (single buffer request => 1)
	byte[] bytes = new byte[bufferSize / 2];
	rand.nextBytes(bytes);

	writer.emit(new ByteArrayIO(bytes));

	// (ii) Larger than the buffer size (two buffer requests => 1 + 2)
	bytes = new byte[bufferSize + 1];
	rand.nextBytes(bytes);

	writer.emit(new ByteArrayIO(bytes));

	// (iii) Exactly the buffer size (single buffer request => 1 + 2 + 1)
	bytes = new byte[bufferSize - lenBytes];
	rand.nextBytes(bytes);

	writer.emit(new ByteArrayIO(bytes));

	// (iv) Nothing on the 4th channel (no buffer request => 1 + 2 + 1 + 0 = 4)

	// (v) Broadcast the event
	writer.broadcastEvent(barrier);

	assertEquals(4, bufferProvider.getNumberOfCreatedBuffers());

	BufferOrEvent boe;
	assertEquals(2, queues[0].size()); // 1 buffer + 1 event
	assertTrue(parseBuffer(queues[0].remove(), 0).isBuffer());
	assertEquals(3, queues[1].size()); // 2 buffers + 1 event
	assertTrue(parseBuffer(queues[1].remove(), 1).isBuffer());
	assertTrue(parseBuffer(queues[1].remove(), 1).isBuffer());
	assertEquals(2, queues[2].size()); // 1 buffer + 1 event
	assertTrue(parseBuffer(queues[2].remove(), 2).isBuffer());
	assertEquals(1, queues[3].size()); // 0 buffers + 1 event

	// every queue's last element should be the event
	for (int i = 0; i < numberOfChannels; i++) {
		boe = parseBuffer(queues[i].remove(), i);
		assertTrue(boe.isEvent());
		assertEquals(barrier, boe.getEvent());
	}
}
 

private static BufferOrEvent createBarrier(long checkpointId, int channel) {
	return new BufferOrEvent(
		new CheckpointBarrier(checkpointId, System.currentTimeMillis(), CheckpointOptions.forCheckpointWithDefaultLocation()),
		new InputChannelInfo(0, channel));
}
 

/**
 * Tests broadcasting events when records have been emitted. The emitted
 * records cover all three {@link SerializationResult} types.
 */
@Test
public void testBroadcastEventMixedRecords() throws Exception {
	Random rand = new XORShiftRandom();
	int numberOfChannels = 4;
	int bufferSize = 32;
	int lenBytes = 4; // serialized length

	@SuppressWarnings("unchecked")
	Queue<BufferConsumer>[] queues = new Queue[numberOfChannels];
	for (int i = 0; i < numberOfChannels; i++) {
		queues[i] = new ArrayDeque<>();
	}

	TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize);

	ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider);
	RecordWriter<ByteArrayIO> writer = new RecordWriterBuilder().build(partitionWriter);
	CheckpointBarrier barrier = new CheckpointBarrier(Integer.MAX_VALUE + 1292L, Integer.MAX_VALUE + 199L, CheckpointOptions.forCheckpointWithDefaultLocation());

	// Emit records on some channels first (requesting buffers), then
	// broadcast the event. The record buffers should be emitted first, then
	// the event. After the event, no new buffer should be requested.

	// (i) Smaller than the buffer size (single buffer request => 1)
	byte[] bytes = new byte[bufferSize / 2];
	rand.nextBytes(bytes);

	writer.emit(new ByteArrayIO(bytes));

	// (ii) Larger than the buffer size (two buffer requests => 1 + 2)
	bytes = new byte[bufferSize + 1];
	rand.nextBytes(bytes);

	writer.emit(new ByteArrayIO(bytes));

	// (iii) Exactly the buffer size (single buffer request => 1 + 2 + 1)
	bytes = new byte[bufferSize - lenBytes];
	rand.nextBytes(bytes);

	writer.emit(new ByteArrayIO(bytes));

	// (iv) Nothing on the 4th channel (no buffer request => 1 + 2 + 1 + 0 = 4)

	// (v) Broadcast the event
	writer.broadcastEvent(barrier);

	assertEquals(4, bufferProvider.getNumberOfCreatedBuffers());

	BufferOrEvent boe;
	assertEquals(2, queues[0].size()); // 1 buffer + 1 event
	assertTrue(parseBuffer(queues[0].remove(), 0).isBuffer());
	assertEquals(3, queues[1].size()); // 2 buffers + 1 event
	assertTrue(parseBuffer(queues[1].remove(), 1).isBuffer());
	assertTrue(parseBuffer(queues[1].remove(), 1).isBuffer());
	assertEquals(2, queues[2].size()); // 1 buffer + 1 event
	assertTrue(parseBuffer(queues[2].remove(), 2).isBuffer());
	assertEquals(1, queues[3].size()); // 0 buffers + 1 event

	// every queue's last element should be the event
	for (int i = 0; i < numberOfChannels; i++) {
		boe = parseBuffer(queues[i].remove(), i);
		assertTrue(boe.isEvent());
		assertEquals(barrier, boe.getEvent());
	}
}
 

private static BufferOrEvent createBarrier(long checkpointId, int channel) {
	return new BufferOrEvent(new CheckpointBarrier(
		checkpointId, System.currentTimeMillis(), CheckpointOptions.forCheckpointWithDefaultLocation()), channel);
}
 

private static BufferOrEvent createBarrier(long id, int channel) {
	return new BufferOrEvent(new CheckpointBarrier(id, System.currentTimeMillis(), CheckpointOptions.forCheckpointWithDefaultLocation()), channel);
}