Java Code Examples for org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.OnCheckpointRollingPolicy#build()

The following examples show how to use org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.OnCheckpointRollingPolicy#build() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HadoopPathBasedBulkFormatBuilder.java    From flink with Apache License 2.0 6 votes vote down vote up 
public HadoopPathBasedBulkFormatBuilder(
	org.apache.hadoop.fs.Path basePath,
	HadoopPathBasedBulkWriter.Factory<IN> writerFactory,
	Configuration configuration,
	BucketAssigner<IN, BucketID> assigner) {

	this(
		basePath,
		writerFactory,
		new DefaultHadoopFileCommitterFactory(),
		configuration,
		assigner,
		OnCheckpointRollingPolicy.build(),
		new DefaultBucketFactoryImpl<>(),
		OutputFileConfig.builder().build());
}
Example 2
Source File: StreamingFileSink.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
Buckets<IN, BucketID> createBuckets(int subtaskIndex) throws IOException {
	return new Buckets<>(
			basePath,
			bucketAssigner,
			bucketFactory,
			new BulkPartWriter.Factory<>(writerFactory),
			OnCheckpointRollingPolicy.build(),
			subtaskIndex);
}
Example 3
Source File: RollingPolicyTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRollOnCheckpointPolicy() throws Exception {
	final File outDir = TEMP_FOLDER.newFolder();
	final Path path = new Path(outDir.toURI());

	final MethodCallCountingPolicyWrapper<String, String> rollingPolicy =
			new MethodCallCountingPolicyWrapper<>(OnCheckpointRollingPolicy.build());

	final Buckets<String, String> buckets = createBuckets(path, rollingPolicy);

	rollingPolicy.verifyCallCounters(0L, 0L, 0L, 0L, 0L, 0L);

	buckets.onElement("test1", new TestUtils.MockSinkContext(1L, 1L, 2L));
	buckets.onElement("test1", new TestUtils.MockSinkContext(2L, 1L, 2L));
	buckets.onElement("test1", new TestUtils.MockSinkContext(3L, 1L, 3L));

	// ... we have a checkpoint so we roll ...
	buckets.snapshotState(1L, new TestUtils.MockListState<>(), new TestUtils.MockListState<>());
	rollingPolicy.verifyCallCounters(1L, 1L, 2L, 0L, 0L, 0L);

	// ... create a new in-progress file (before we had closed the last one so it was null)...
	buckets.onElement("test1", new TestUtils.MockSinkContext(5L, 1L, 5L));

	// ... we have a checkpoint so we roll ...
	buckets.snapshotState(2L, new TestUtils.MockListState<>(), new TestUtils.MockListState<>());
	rollingPolicy.verifyCallCounters(2L, 2L, 2L, 0L, 0L, 0L);

	buckets.close();
}
Example 4
Source File: StreamingFileSink.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
Buckets<IN, BucketID> createBuckets(int subtaskIndex) throws IOException {
	return new Buckets<>(
			basePath,
			bucketAssigner,
			bucketFactory,
			new BulkPartWriter.Factory<>(writerFactory),
			OnCheckpointRollingPolicy.build(),
			subtaskIndex,
			new PartFileConfig(partFilePrefix, partFileSuffix));
}
Example 5
Source File: RollingPolicyTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRollOnCheckpointPolicy() throws Exception {
	final File outDir = TEMP_FOLDER.newFolder();
	final Path path = new Path(outDir.toURI());

	final MethodCallCountingPolicyWrapper<String, String> rollingPolicy =
			new MethodCallCountingPolicyWrapper<>(OnCheckpointRollingPolicy.build());

	final Buckets<String, String> buckets = createBuckets(path, rollingPolicy);

	rollingPolicy.verifyCallCounters(0L, 0L, 0L, 0L, 0L, 0L);

	buckets.onElement("test1", new TestUtils.MockSinkContext(1L, 1L, 2L));
	buckets.onElement("test1", new TestUtils.MockSinkContext(2L, 1L, 2L));
	buckets.onElement("test1", new TestUtils.MockSinkContext(3L, 1L, 3L));

	// ... we have a checkpoint so we roll ...
	buckets.snapshotState(1L, new TestUtils.MockListState<>(), new TestUtils.MockListState<>());
	rollingPolicy.verifyCallCounters(1L, 1L, 2L, 0L, 0L, 0L);

	// ... create a new in-progress file (before we had closed the last one so it was null)...
	buckets.onElement("test1", new TestUtils.MockSinkContext(5L, 1L, 5L));

	// ... we have a checkpoint so we roll ...
	buckets.snapshotState(2L, new TestUtils.MockListState<>(), new TestUtils.MockListState<>());
	rollingPolicy.verifyCallCounters(2L, 2L, 2L, 0L, 0L, 0L);

	buckets.close();
}
Example 6
Source File: RollingPolicyTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRollOnCheckpointPolicy() throws Exception {
	final File outDir = TEMP_FOLDER.newFolder();
	final Path path = new Path(outDir.toURI());

	final MethodCallCountingPolicyWrapper<String, String> rollingPolicy =
			new MethodCallCountingPolicyWrapper<>(OnCheckpointRollingPolicy.build());

	final Buckets<String, String> buckets = createBuckets(path, rollingPolicy);

	rollingPolicy.verifyCallCounters(0L, 0L, 0L, 0L, 0L, 0L);

	buckets.onElement("test1", new TestUtils.MockSinkContext(1L, 1L, 2L));
	buckets.onElement("test1", new TestUtils.MockSinkContext(2L, 1L, 2L));
	buckets.onElement("test1", new TestUtils.MockSinkContext(3L, 1L, 3L));

	// ... we have a checkpoint so we roll ...
	buckets.snapshotState(1L, new TestUtils.MockListState<>(), new TestUtils.MockListState<>());
	rollingPolicy.verifyCallCounters(1L, 1L, 2L, 0L, 0L, 0L);

	// ... create a new in-progress file (before we had closed the last one so it was null)...
	buckets.onElement("test1", new TestUtils.MockSinkContext(5L, 1L, 5L));

	// ... we have a checkpoint so we roll ...
	buckets.snapshotState(2L, new TestUtils.MockListState<>(), new TestUtils.MockListState<>());
	rollingPolicy.verifyCallCounters(2L, 2L, 2L, 0L, 0L, 0L);

	buckets.close();
}
Example 7
Source File: StreamingFileSink.java    From flink with Apache License 2.0 4 votes vote down vote up 
protected BulkFormatBuilder(Path basePath, BulkWriter.Factory<IN> writerFactory, BucketAssigner<IN, BucketID> assigner) {
	this(basePath, writerFactory, assigner, OnCheckpointRollingPolicy.build(), DEFAULT_BUCKET_CHECK_INTERVAL,
		new DefaultBucketFactoryImpl<>(), OutputFileConfig.builder().build());
}
Example 8
Source File: BucketsTest.java    From Flink-CEPplus with Apache License 2.0 3 votes vote down vote up 
@Test
public void testSnapshotAndRestore() throws Exception {
	final File outDir = TEMP_FOLDER.newFolder();
	final Path path = new Path(outDir.toURI());

	final RollingPolicy<String, String> onCheckpointRollingPolicy = OnCheckpointRollingPolicy.build();

	final Buckets<String, String> buckets = createBuckets(path, onCheckpointRollingPolicy, 0);

	final ListState<byte[]> bucketStateContainer = new MockListState<>();
	final ListState<Long> partCounterContainer = new MockListState<>();

	buckets.onElement("test1", new TestUtils.MockSinkContext(null, 1L, 2L));
	buckets.snapshotState(0L, bucketStateContainer, partCounterContainer);

	assertThat(buckets.getActiveBuckets().get("test1"), hasSinglePartFileToBeCommittedOnCheckpointAck(path, "test1"));

	buckets.onElement("test2", new TestUtils.MockSinkContext(null, 1L, 2L));
	buckets.snapshotState(1L, bucketStateContainer, partCounterContainer);

	assertThat(buckets.getActiveBuckets().get("test1"), hasSinglePartFileToBeCommittedOnCheckpointAck(path, "test1"));
	assertThat(buckets.getActiveBuckets().get("test2"), hasSinglePartFileToBeCommittedOnCheckpointAck(path, "test2"));

	Buckets<String, String> restoredBuckets =
			restoreBuckets(path, onCheckpointRollingPolicy, 0, bucketStateContainer, partCounterContainer);

	final Map<String, Bucket<String, String>> activeBuckets = restoredBuckets.getActiveBuckets();

	// because we commit pending files for previous checkpoints upon recovery
	Assert.assertTrue(activeBuckets.isEmpty());
}
Example 9
Source File: BucketsTest.java    From flink with Apache License 2.0 3 votes vote down vote up 
@Test
public void testSnapshotAndRestore() throws Exception {
	final File outDir = TEMP_FOLDER.newFolder();
	final Path path = new Path(outDir.toURI());

	final RollingPolicy<String, String> onCheckpointRollingPolicy = OnCheckpointRollingPolicy.build();

	final Buckets<String, String> buckets = createBuckets(path, onCheckpointRollingPolicy, 0);

	final ListState<byte[]> bucketStateContainer = new MockListState<>();
	final ListState<Long> partCounterContainer = new MockListState<>();

	buckets.onElement("test1", new TestUtils.MockSinkContext(null, 1L, 2L));
	buckets.snapshotState(0L, bucketStateContainer, partCounterContainer);

	assertThat(buckets.getActiveBuckets().get("test1"), hasSinglePartFileToBeCommittedOnCheckpointAck(path, "test1"));

	buckets.onElement("test2", new TestUtils.MockSinkContext(null, 1L, 2L));
	buckets.snapshotState(1L, bucketStateContainer, partCounterContainer);

	assertThat(buckets.getActiveBuckets().get("test1"), hasSinglePartFileToBeCommittedOnCheckpointAck(path, "test1"));
	assertThat(buckets.getActiveBuckets().get("test2"), hasSinglePartFileToBeCommittedOnCheckpointAck(path, "test2"));

	Buckets<String, String> restoredBuckets =
			restoreBuckets(path, onCheckpointRollingPolicy, 0, bucketStateContainer, partCounterContainer);

	final Map<String, Bucket<String, String>> activeBuckets = restoredBuckets.getActiveBuckets();

	// because we commit pending files for previous checkpoints upon recovery
	Assert.assertTrue(activeBuckets.isEmpty());
}
Example 10
Source File: BucketsTest.java    From flink with Apache License 2.0 3 votes vote down vote up 
@Test
public void testSnapshotAndRestore() throws Exception {
	final File outDir = TEMP_FOLDER.newFolder();
	final Path path = new Path(outDir.toURI());

	final RollingPolicy<String, String> onCheckpointRollingPolicy = OnCheckpointRollingPolicy.build();

	final Buckets<String, String> buckets = createBuckets(path, onCheckpointRollingPolicy, 0);

	final ListState<byte[]> bucketStateContainer = new MockListState<>();
	final ListState<Long> partCounterContainer = new MockListState<>();

	buckets.onElement("test1", new TestUtils.MockSinkContext(null, 1L, 2L));
	buckets.snapshotState(0L, bucketStateContainer, partCounterContainer);

	assertThat(buckets.getActiveBuckets().get("test1"), hasSinglePartFileToBeCommittedOnCheckpointAck(path, "test1"));

	buckets.onElement("test2", new TestUtils.MockSinkContext(null, 1L, 2L));
	buckets.snapshotState(1L, bucketStateContainer, partCounterContainer);

	assertThat(buckets.getActiveBuckets().get("test1"), hasSinglePartFileToBeCommittedOnCheckpointAck(path, "test1"));
	assertThat(buckets.getActiveBuckets().get("test2"), hasSinglePartFileToBeCommittedOnCheckpointAck(path, "test2"));

	Buckets<String, String> restoredBuckets =
			restoreBuckets(path, onCheckpointRollingPolicy, 0, bucketStateContainer, partCounterContainer);

	final Map<String, Bucket<String, String>> activeBuckets = restoredBuckets.getActiveBuckets();

	// because we commit pending files for previous checkpoints upon recovery
	Assert.assertTrue(activeBuckets.isEmpty());
}