org.apache.flink.runtime.state.filesystem.FsStateBackend Java Examples
The following examples show how to use
org.apache.flink.runtime.state.filesystem.FsStateBackend.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: StatefulStreamingJob.java From flink with Apache License 2.0 | 6 votes |
|
public static void main(String[] args) throws Exception {
final ParameterTool pt = ParameterTool.fromArgs(args);
final String checkpointDir = pt.getRequired("checkpoint.dir");
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setStateBackend(new FsStateBackend(checkpointDir));
env.setRestartStrategy(RestartStrategies.noRestart());
env.enableCheckpointing(1000L);
env.getConfig().disableGenericTypes();
env.addSource(new MySource()).uid("my-source")
.keyBy(anInt -> 0)
.map(new MyStatefulFunction()).uid("my-map")
.addSink(new DiscardingSink<>()).uid("my-sink");
env.execute();
}
Example #2
| Source File: RocksDBTtlStateTestBase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
StateBackend createStateBackend(TernaryBoolean enableIncrementalCheckpointing) {
String dbPath;
String checkpointPath;
try {
dbPath = tempFolder.newFolder().getAbsolutePath();
checkpointPath = tempFolder.newFolder().toURI().toString();
} catch (IOException e) {
throw new FlinkRuntimeException("Failed to init rocksdb test state backend");
}
RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
Configuration config = new Configuration();
config.setBoolean(TTL_COMPACT_FILTER_ENABLED, true);
backend = backend.configure(config, Thread.currentThread().getContextClassLoader());
backend.setDbStoragePath(dbPath);
return backend;
}
Example #3
| Source File: DataStreamAllroundTestJobFactory.java From flink with Apache License 2.0 | 6 votes |
|
private static void setupStateBackend(final StreamExecutionEnvironment env, final ParameterTool pt) throws IOException {
final String stateBackend = pt.get(
STATE_BACKEND.key(),
STATE_BACKEND.defaultValue());
final String checkpointDir = pt.getRequired(STATE_BACKEND_CHECKPOINT_DIR.key());
if ("file".equalsIgnoreCase(stateBackend)) {
boolean asyncCheckpoints = pt.getBoolean(
STATE_BACKEND_FILE_ASYNC.key(),
STATE_BACKEND_FILE_ASYNC.defaultValue());
env.setStateBackend((StateBackend) new FsStateBackend(checkpointDir, asyncCheckpoints));
} else if ("rocks".equalsIgnoreCase(stateBackend)) {
boolean incrementalCheckpoints = pt.getBoolean(
STATE_BACKEND_ROCKS_INCREMENTAL.key(),
STATE_BACKEND_ROCKS_INCREMENTAL.defaultValue());
env.setStateBackend((StateBackend) new RocksDBStateBackend(checkpointDir, incrementalCheckpoints));
} else {
throw new IllegalArgumentException("Unknown backend requested: " + stateBackend);
}
}
Example #4
| Source File: StatefulStreamingJob.java From flink with Apache License 2.0 | 6 votes |
|
public static void main(String[] args) throws Exception {
final ParameterTool pt = ParameterTool.fromArgs(args);
final String checkpointDir = pt.getRequired("checkpoint.dir");
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setStateBackend(new FsStateBackend(checkpointDir));
env.setRestartStrategy(RestartStrategies.noRestart());
env.enableCheckpointing(1000L);
env.getConfig().disableGenericTypes();
env.addSource(new MySource()).uid("my-source")
.keyBy(anInt -> 0)
.map(new MyStatefulFunction()).uid("my-map")
.addSink(new DiscardingSink<>()).uid("my-sink");
env.execute();
}
Example #5
| Source File: StatefulStreamingJob.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
public static void main(String[] args) throws Exception {
final ParameterTool pt = ParameterTool.fromArgs(args);
final String checkpointDir = pt.getRequired("checkpoint.dir");
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setStateBackend(new FsStateBackend(checkpointDir));
env.setRestartStrategy(RestartStrategies.noRestart());
env.enableCheckpointing(1000L);
env.getConfig().disableGenericTypes();
env.addSource(new MySource()).uid("my-source")
.keyBy(anInt -> 0)
.map(new MyStatefulFunction()).uid("my-map")
.addSink(new DiscardingSink<>()).uid("my-sink");
env.execute();
}
Example #6
| Source File: EventTimeWindowCheckpointingITCase.java From flink with Apache License 2.0 | 6 votes |
|
private void setupRocksDB(Configuration config, int fileSizeThreshold, boolean incrementalCheckpoints) throws IOException {
// Configure the managed memory size as 64MB per slot for rocksDB state backend.
config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.ofMebiBytes(PARALLELISM / NUM_OF_TASK_MANAGERS * 64));
String rocksDb = tempFolder.newFolder().getAbsolutePath();
String backups = tempFolder.newFolder().getAbsolutePath();
// we use the fs backend with small threshold here to test the behaviour with file
// references, not self contained byte handles
RocksDBStateBackend rdb =
new RocksDBStateBackend(
new FsStateBackend(
new Path("file://" + backups).toUri(), fileSizeThreshold),
incrementalCheckpoints);
rdb.setDbStoragePath(rocksDb);
this.stateBackend = rdb;
}
Example #7
| Source File: CassandraTupleWriteAheadSinkExample.java From flink-learning with Apache License 2.0 | 6 votes |
|
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.enableCheckpointing(1000);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
env.setStateBackend(new FsStateBackend("file:///" + System.getProperty("java.io.tmpdir") + "/flink/backend"));
CassandraSink<Tuple2<String, Integer>> sink = CassandraSink.addSink(env.addSource(new MySource()))
.setQuery("INSERT INTO zhisheng.values (id, counter) values (?, ?);")
.enableWriteAheadLog()
.setClusterBuilder(new ClusterBuilder() {
private static final long serialVersionUID = 2793938419775311824L;
@Override
public Cluster buildCluster(Cluster.Builder builder) {
return builder.addContactPoint("127.0.0.1").build();
}
})
.build();
sink.name("Cassandra Sink").disableChaining().setParallelism(1).uid("hello");
env.execute();
}
Example #8
| Source File: CassandraTupleWriteAheadSinkExample.java From flink with Apache License 2.0 | 6 votes |
|
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.enableCheckpointing(1000);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
env.setStateBackend(new FsStateBackend("file:///" + System.getProperty("java.io.tmpdir") + "/flink/backend"));
CassandraSink<Tuple2<String, Integer>> sink = CassandraSink.addSink(env.addSource(new MySource()))
.setQuery("INSERT INTO example.values (id, counter) values (?, ?);")
.enableWriteAheadLog()
.setClusterBuilder(new ClusterBuilder() {
private static final long serialVersionUID = 2793938419775311824L;
@Override
public Cluster buildCluster(Cluster.Builder builder) {
return builder.addContactPoint("127.0.0.1").build();
}
})
.build();
sink.name("Cassandra Sink").disableChaining().setParallelism(1).uid("hello");
env.execute();
}
Example #9
| Source File: CassandraTupleWriteAheadSinkExample.java From flink with Apache License 2.0 | 6 votes |
|
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.enableCheckpointing(1000);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
env.setStateBackend(new FsStateBackend("file:///" + System.getProperty("java.io.tmpdir") + "/flink/backend"));
CassandraSink<Tuple2<String, Integer>> sink = CassandraSink.addSink(env.addSource(new MySource()))
.setQuery("INSERT INTO example.values (id, counter) values (?, ?);")
.enableWriteAheadLog()
.setClusterBuilder(new ClusterBuilder() {
private static final long serialVersionUID = 2793938419775311824L;
@Override
public Cluster buildCluster(Cluster.Builder builder) {
return builder.addContactPoint("127.0.0.1").build();
}
})
.build();
sink.name("Cassandra Sink").disableChaining().setParallelism(1).uid("hello");
env.execute();
}
Example #10
| Source File: AdvertisingTopologyFlinkStateHighKeyCard.java From yahoo-streaming-benchmark with Apache License 2.0 | 6 votes |
|
/**
* Do some Flink Configuration
*/
private static StreamExecutionEnvironment setupFlinkEnvironment(BenchmarkConfig config) throws IOException {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.getConfig().setGlobalJobParameters(config.getParameters());
env.getConfig().enableObjectReuse();
// enable checkpointing for fault tolerance
if (config.checkpointsEnabled) {
env.enableCheckpointing(config.checkpointInterval);
if (config.checkpointToUri) {
env.setStateBackend(new FsStateBackend(config.checkpointUri));
}
}
// use event time
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
return env;
}
Example #11
| Source File: RocksDBTtlStateTestBase.java From flink with Apache License 2.0 | 6 votes |
|
StateBackend createStateBackend(TernaryBoolean enableIncrementalCheckpointing) {
String dbPath;
String checkpointPath;
try {
dbPath = tempFolder.newFolder().getAbsolutePath();
checkpointPath = tempFolder.newFolder().toURI().toString();
} catch (IOException e) {
throw new FlinkRuntimeException("Failed to init rocksdb test state backend");
}
RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
Configuration config = new Configuration();
config.setBoolean(TTL_COMPACT_FILTER_ENABLED, true);
backend = backend.configure(config, Thread.currentThread().getContextClassLoader());
backend.setDbStoragePath(dbPath);
return backend;
}
Example #12
| Source File: CassandraTupleWriteAheadSinkExample.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.enableCheckpointing(1000);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
env.setStateBackend(new FsStateBackend("file:///" + System.getProperty("java.io.tmpdir") + "/flink/backend"));
CassandraSink<Tuple2<String, Integer>> sink = CassandraSink.addSink(env.addSource(new MySource()))
.setQuery("INSERT INTO example.values (id, counter) values (?, ?);")
.enableWriteAheadLog()
.setClusterBuilder(new ClusterBuilder() {
private static final long serialVersionUID = 2793938419775311824L;
@Override
public Cluster buildCluster(Cluster.Builder builder) {
return builder.addContactPoint("127.0.0.1").build();
}
})
.build();
sink.name("Cassandra Sink").disableChaining().setParallelism(1).uid("hello");
env.execute();
}
Example #13
| Source File: CassandraTupleWriteAheadSinkExample.java From flink-learning with Apache License 2.0 | 6 votes |
|
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.enableCheckpointing(1000);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
env.setStateBackend(new FsStateBackend("file:///" + System.getProperty("java.io.tmpdir") + "/flink/backend"));
CassandraSink<Tuple2<String, Integer>> sink = CassandraSink.addSink(env.addSource(new MySource()))
.setQuery("INSERT INTO zhisheng.values (id, counter) values (?, ?);")
.enableWriteAheadLog()
.setClusterBuilder(new ClusterBuilder() {
private static final long serialVersionUID = 2793938419775311824L;
@Override
public Cluster buildCluster(Cluster.Builder builder) {
return builder.addContactPoint("127.0.0.1").build();
}
})
.build();
sink.name("Cassandra Sink").disableChaining().setParallelism(1).uid("hello");
env.execute();
}
Example #14
| Source File: DataStreamAllroundTestJobFactory.java From flink with Apache License 2.0 | 6 votes |
|
private static void setupStateBackend(final StreamExecutionEnvironment env, final ParameterTool pt) throws IOException {
final String stateBackend = pt.get(
STATE_BACKEND.key(),
STATE_BACKEND.defaultValue());
final String checkpointDir = pt.getRequired(STATE_BACKEND_CHECKPOINT_DIR.key());
if ("file".equalsIgnoreCase(stateBackend)) {
boolean asyncCheckpoints = pt.getBoolean(
STATE_BACKEND_FILE_ASYNC.key(),
STATE_BACKEND_FILE_ASYNC.defaultValue());
env.setStateBackend((StateBackend) new FsStateBackend(checkpointDir, asyncCheckpoints));
} else if ("rocks".equalsIgnoreCase(stateBackend)) {
boolean incrementalCheckpoints = pt.getBoolean(
STATE_BACKEND_ROCKS_INCREMENTAL.key(),
STATE_BACKEND_ROCKS_INCREMENTAL.defaultValue());
env.setStateBackend((StateBackend) new RocksDBStateBackend(checkpointDir, incrementalCheckpoints));
} else {
throw new IllegalArgumentException("Unknown backend requested: " + stateBackend);
}
}
Example #15
| Source File: RocksDBStateBackendMigrationTest.java From flink with Apache License 2.0 | 5 votes |
|
@Override
protected RocksDBStateBackend getStateBackend() throws IOException {
dbPath = tempFolder.newFolder().getAbsolutePath();
String checkpointPath = tempFolder.newFolder().toURI().toString();
RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
Configuration configuration = new Configuration();
configuration.set(RocksDBOptions.TIMER_SERVICE_FACTORY, RocksDBStateBackend.PriorityQueueStateType.ROCKSDB);
backend = backend.configure(configuration, Thread.currentThread().getContextClassLoader());
backend.setDbStoragePath(dbPath);
return backend;
}
Example #16
| Source File: HeapKeyedStateBackendAsyncByDefaultTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testFsStateBackendDefaultsToAsync() throws Exception {
FsStateBackend backend = new FsStateBackend(tmpFolder.newFolder().toURI());
assertTrue(backend.isUsingAsynchronousSnapshots());
validateSupportForAsyncSnapshots(backend);
}
Example #17
| Source File: RocksDBStateBackendTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Override
protected RocksDBStateBackend getStateBackend() throws IOException {
dbPath = tempFolder.newFolder().getAbsolutePath();
String checkpointPath = tempFolder.newFolder().toURI().toString();
RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
Configuration configuration = new Configuration();
configuration.setString(
RocksDBOptions.TIMER_SERVICE_FACTORY,
RocksDBStateBackend.PriorityQueueStateType.ROCKSDB.toString());
backend = backend.configure(configuration, Thread.currentThread().getContextClassLoader());
backend.setDbStoragePath(dbPath);
return backend;
}
Example #18
| Source File: RocksDBStateBackendMigrationTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Override
protected RocksDBStateBackend getStateBackend() throws IOException {
dbPath = tempFolder.newFolder().getAbsolutePath();
String checkpointPath = tempFolder.newFolder().toURI().toString();
RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
Configuration configuration = new Configuration();
configuration.setString(
RocksDBOptions.TIMER_SERVICE_FACTORY,
RocksDBStateBackend.PriorityQueueStateType.ROCKSDB.toString());
backend = backend.configure(configuration, Thread.currentThread().getContextClassLoader());
backend.setDbStoragePath(dbPath);
return backend;
}
Example #19
| Source File: RocksDBTtlStateTestBase.java From flink with Apache License 2.0 | 5 votes |
|
StateBackend createStateBackend(TernaryBoolean enableIncrementalCheckpointing) {
String dbPath;
String checkpointPath;
try {
dbPath = tempFolder.newFolder().getAbsolutePath();
checkpointPath = tempFolder.newFolder().toURI().toString();
} catch (IOException e) {
throw new FlinkRuntimeException("Failed to init rocksdb test state backend");
}
RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
Configuration config = new Configuration();
backend = backend.configure(config, Thread.currentThread().getContextClassLoader());
backend.setDbStoragePath(dbPath);
return backend;
}
Example #20
| Source File: RocksDBStateBackendConfigTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testRocksDbReconfigurationCopiesExistingValues() throws Exception {
final FsStateBackend checkpointBackend = new FsStateBackend(tempFolder.newFolder().toURI().toString());
final boolean incremental = !CheckpointingOptions.INCREMENTAL_CHECKPOINTS.defaultValue();
final RocksDBStateBackend original = new RocksDBStateBackend(checkpointBackend, incremental);
// these must not be the default options
final PredefinedOptions predOptions = PredefinedOptions.SPINNING_DISK_OPTIMIZED_HIGH_MEM;
assertNotEquals(predOptions, original.getPredefinedOptions());
original.setPredefinedOptions(predOptions);
final RocksDBOptionsFactory optionsFactory = mock(RocksDBOptionsFactory.class);
original.setRocksDBOptions(optionsFactory);
final String[] localDirs = new String[] {
tempFolder.newFolder().getAbsolutePath(), tempFolder.newFolder().getAbsolutePath() };
original.setDbStoragePaths(localDirs);
RocksDBStateBackend copy = original.configure(new Configuration(), Thread.currentThread().getContextClassLoader());
assertEquals(original.isIncrementalCheckpointsEnabled(), copy.isIncrementalCheckpointsEnabled());
assertArrayEquals(original.getDbStoragePaths(), copy.getDbStoragePaths());
assertEquals(original.getRocksDBOptions(), copy.getRocksDBOptions());
assertEquals(original.getPredefinedOptions(), copy.getPredefinedOptions());
FsStateBackend copyCheckpointBackend = (FsStateBackend) copy.getCheckpointBackend();
assertEquals(checkpointBackend.getCheckpointPath(), copyCheckpointBackend.getCheckpointPath());
assertEquals(checkpointBackend.getSavepointPath(), copyCheckpointBackend.getSavepointPath());
}
Example #21
| Source File: EventTimeWindowCheckpointingITCase.java From flink with Apache License 2.0 | 5 votes |
|
private void setupRocksDB(int fileSizeThreshold, boolean incrementalCheckpoints) throws IOException {
String rocksDb = tempFolder.newFolder().getAbsolutePath();
String backups = tempFolder.newFolder().getAbsolutePath();
// we use the fs backend with small threshold here to test the behaviour with file
// references, not self contained byte handles
RocksDBStateBackend rdb =
new RocksDBStateBackend(
new FsStateBackend(
new Path("file://" + backups).toUri(), fileSizeThreshold),
incrementalCheckpoints);
rdb.setDbStoragePath(rocksDb);
this.stateBackend = rdb;
}
Example #22
| Source File: HeapKeyedStateBackendAsyncByDefaultTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testFsStateBackendDefaultsToAsync() throws Exception {
FsStateBackend backend = new FsStateBackend(tmpFolder.newFolder().toURI());
assertTrue(backend.isUsingAsynchronousSnapshots());
validateSupportForAsyncSnapshots(backend);
}
Example #23
| Source File: CheckpointedLongRidesSolution.java From flink-training-exercises with Apache License 2.0 | 5 votes |
|
public static void main(String[] args) throws Exception {
ParameterTool params = ParameterTool.fromArgs(args);
final String input = params.get("input", ExerciseBase.pathToRideData);
final int servingSpeedFactor = 1800; // 30 minutes worth of events are served every second
// set up streaming execution environment
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.setParallelism(ExerciseBase.parallelism);
// set up checkpointing
env.setStateBackend(new FsStateBackend("file:///tmp/checkpoints"));
env.enableCheckpointing(1000);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(60, Time.of(10, TimeUnit.SECONDS)));
DataStream<TaxiRide> rides = env.addSource(rideSourceOrTest(new CheckpointedTaxiRideSource(input, servingSpeedFactor)));
DataStream<TaxiRide> longRides = rides
.filter(new NYCFilter())
.keyBy((TaxiRide ride) -> ride.rideId)
.process(new MatchFunction());
printOrTest(longRides);
env.execute("Long Taxi Rides (checkpointed)");
}
Example #24
| Source File: RocksDBStateBackendTest.java From flink with Apache License 2.0 | 5 votes |
|
@Override
protected RocksDBStateBackend getStateBackend() throws IOException {
dbPath = tempFolder.newFolder().getAbsolutePath();
String checkpointPath = tempFolder.newFolder().toURI().toString();
RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
Configuration configuration = new Configuration();
configuration.setString(
RocksDBOptions.TIMER_SERVICE_FACTORY,
RocksDBStateBackend.PriorityQueueStateType.ROCKSDB.toString());
backend = backend.configure(configuration, Thread.currentThread().getContextClassLoader());
backend.setDbStoragePath(dbPath);
return backend;
}
Example #25
| Source File: RocksDBStateBackendMigrationTest.java From flink with Apache License 2.0 | 5 votes |
|
@Override
protected RocksDBStateBackend getStateBackend() throws IOException {
dbPath = tempFolder.newFolder().getAbsolutePath();
String checkpointPath = tempFolder.newFolder().toURI().toString();
RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
Configuration configuration = new Configuration();
configuration.setString(
RocksDBOptions.TIMER_SERVICE_FACTORY,
RocksDBStateBackend.PriorityQueueStateType.ROCKSDB.toString());
backend = backend.configure(configuration, Thread.currentThread().getContextClassLoader());
backend.setDbStoragePath(dbPath);
return backend;
}
Example #26
| Source File: RocksDBStateBackendConfigTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testRocksDbReconfigurationCopiesExistingValues() throws Exception {
final FsStateBackend checkpointBackend = new FsStateBackend(tempFolder.newFolder().toURI().toString());
final boolean incremental = !CheckpointingOptions.INCREMENTAL_CHECKPOINTS.defaultValue();
final RocksDBStateBackend original = new RocksDBStateBackend(checkpointBackend, incremental);
// these must not be the default options
final PredefinedOptions predOptions = PredefinedOptions.SPINNING_DISK_OPTIMIZED_HIGH_MEM;
assertNotEquals(predOptions, original.getPredefinedOptions());
original.setPredefinedOptions(predOptions);
final OptionsFactory optionsFactory = mock(OptionsFactory.class);
original.setOptions(optionsFactory);
final String[] localDirs = new String[] {
tempFolder.newFolder().getAbsolutePath(), tempFolder.newFolder().getAbsolutePath() };
original.setDbStoragePaths(localDirs);
RocksDBStateBackend copy = original.configure(new Configuration(), Thread.currentThread().getContextClassLoader());
assertEquals(original.isIncrementalCheckpointsEnabled(), copy.isIncrementalCheckpointsEnabled());
assertArrayEquals(original.getDbStoragePaths(), copy.getDbStoragePaths());
assertEquals(original.getOptions(), copy.getOptions());
assertEquals(original.getPredefinedOptions(), copy.getPredefinedOptions());
FsStateBackend copyCheckpointBackend = (FsStateBackend) copy.getCheckpointBackend();
assertEquals(checkpointBackend.getCheckpointPath(), copyCheckpointBackend.getCheckpointPath());
assertEquals(checkpointBackend.getSavepointPath(), copyCheckpointBackend.getSavepointPath());
}
Example #27
| Source File: EventTimeWindowCheckpointingITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
private void setupRocksDB(int fileSizeThreshold, boolean incrementalCheckpoints) throws IOException {
String rocksDb = tempFolder.newFolder().getAbsolutePath();
String backups = tempFolder.newFolder().getAbsolutePath();
// we use the fs backend with small threshold here to test the behaviour with file
// references, not self contained byte handles
RocksDBStateBackend rdb =
new RocksDBStateBackend(
new FsStateBackend(
new Path("file://" + backups).toUri(), fileSizeThreshold),
incrementalCheckpoints);
rdb.setDbStoragePath(rocksDb);
this.stateBackend = rdb;
}
Example #28
| Source File: HeapKeyedStateBackendAsyncByDefaultTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testFsStateBackendDefaultsToAsync() throws Exception {
FsStateBackend backend = new FsStateBackend(tmpFolder.newFolder().toURI());
assertTrue(backend.isUsingAsynchronousSnapshots());
validateSupportForAsyncSnapshots(backend);
}
Example #29
| Source File: StateBackendLoadingTest.java From flink with Apache License 2.0 | 5 votes |
|
/**
* Validates taking the application-defined file system state backend and adding with additional
* parameters from the cluster configuration, but giving precedence to application-defined
* parameters over configuration-defined parameters.
*/
@Test
public void testLoadFileSystemStateBackendMixed() throws Exception {
final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString();
final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
final String savepointDir = new Path(tmp.newFolder().toURI()).toString();
final Path expectedCheckpointsPath = new Path(new URI(appCheckpointDir));
final Path expectedSavepointsPath = new Path(savepointDir);
final int threshold = 1000000;
final int writeBufferSize = 4000000;
final FsStateBackend backend = new FsStateBackend(new URI(appCheckpointDir), null, threshold, writeBufferSize, TernaryBoolean.TRUE);
final Configuration config = new Configuration();
config.setString(backendKey, "jobmanager"); // this should not be picked up
config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this should not be picked up
config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
config.setInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, 20); // this should not be picked up
config.setInteger(CheckpointingOptions.FS_WRITE_BUFFER_SIZE, 3000000); // this should not be picked up
final StateBackend loadedBackend =
StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null);
assertTrue(loadedBackend instanceof FsStateBackend);
final FsStateBackend fs = (FsStateBackend) loadedBackend;
assertEquals(expectedCheckpointsPath, fs.getCheckpointPath());
assertEquals(expectedSavepointsPath, fs.getSavepointPath());
assertEquals(threshold, fs.getMinFileSizeThreshold());
assertEquals(writeBufferSize, fs.getWriteBufferSize());
}
Example #30
| Source File: RocksDBStateBackendConfigTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testRocksDbReconfigurationCopiesExistingValues() throws Exception {
final FsStateBackend checkpointBackend = new FsStateBackend(tempFolder.newFolder().toURI().toString());
final boolean incremental = !CheckpointingOptions.INCREMENTAL_CHECKPOINTS.defaultValue();
final RocksDBStateBackend original = new RocksDBStateBackend(checkpointBackend, incremental);
// these must not be the default options
final PredefinedOptions predOptions = PredefinedOptions.SPINNING_DISK_OPTIMIZED_HIGH_MEM;
assertNotEquals(predOptions, original.getPredefinedOptions());
original.setPredefinedOptions(predOptions);
final OptionsFactory optionsFactory = mock(OptionsFactory.class);
original.setOptions(optionsFactory);
final String[] localDirs = new String[] {
tempFolder.newFolder().getAbsolutePath(), tempFolder.newFolder().getAbsolutePath() };
original.setDbStoragePaths(localDirs);
RocksDBStateBackend copy = original.configure(new Configuration(), Thread.currentThread().getContextClassLoader());
assertEquals(original.isIncrementalCheckpointsEnabled(), copy.isIncrementalCheckpointsEnabled());
assertArrayEquals(original.getDbStoragePaths(), copy.getDbStoragePaths());
assertEquals(original.getOptions(), copy.getOptions());
assertEquals(original.getPredefinedOptions(), copy.getPredefinedOptions());
FsStateBackend copyCheckpointBackend = (FsStateBackend) copy.getCheckpointBackend();
assertEquals(checkpointBackend.getCheckpointPath(), copyCheckpointBackend.getCheckpointPath());
assertEquals(checkpointBackend.getSavepointPath(), copyCheckpointBackend.getSavepointPath());
}
