org.apache.flink.util.TernaryBoolean Java Examples
The following examples show how to use
org.apache.flink.util.TernaryBoolean.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RocksDBTtlStateTestBase.java From flink with Apache License 2.0 | 6 votes |
StateBackend createStateBackend(TernaryBoolean enableIncrementalCheckpointing) { String dbPath; String checkpointPath; try { dbPath = tempFolder.newFolder().getAbsolutePath(); checkpointPath = tempFolder.newFolder().toURI().toString(); } catch (IOException e) { throw new FlinkRuntimeException("Failed to init rocksdb test state backend"); } RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing); Configuration config = new Configuration(); config.setBoolean(TTL_COMPACT_FILTER_ENABLED, true); backend = backend.configure(config, Thread.currentThread().getContextClassLoader()); backend.setDbStoragePath(dbPath); return backend; }
Example #2
Source File: RocksDBTtlStateTestBase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
StateBackend createStateBackend(TernaryBoolean enableIncrementalCheckpointing) { String dbPath; String checkpointPath; try { dbPath = tempFolder.newFolder().getAbsolutePath(); checkpointPath = tempFolder.newFolder().toURI().toString(); } catch (IOException e) { throw new FlinkRuntimeException("Failed to init rocksdb test state backend"); } RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing); Configuration config = new Configuration(); config.setBoolean(TTL_COMPACT_FILTER_ENABLED, true); backend = backend.configure(config, Thread.currentThread().getContextClassLoader()); backend.setDbStoragePath(dbPath); return backend; }
Example #3
Source File: CsvTableSinkFactoryTest.java From flink with Apache License 2.0 | 6 votes |
private DescriptorProperties createDescriptor(TableSchema schema) { Map<String, String> properties = new HashMap<>(); properties.put("connector.type", "filesystem"); properties.put("connector.property-version", "1"); properties.put("connector.path", "/path/to/csv"); // schema properties.put("format.type", "csv"); properties.put("format.property-version", "1"); properties.put("format.field-delimiter", ";"); DescriptorProperties descriptor = new DescriptorProperties(true); descriptor.putProperties(properties); descriptor.putTableSchema(SCHEMA, schema); if (deriveSchema == TernaryBoolean.TRUE) { descriptor.putBoolean("format.derive-schema", true); } else if (deriveSchema == TernaryBoolean.FALSE) { descriptor.putTableSchema(FORMAT_FIELDS, testingSchema); } // nothing to put for UNDEFINED return descriptor; }
Example #4
Source File: StateBackendLoadingTest.java From flink with Apache License 2.0 | 5 votes |
/** * Validates taking the application-defined file system state backend and adding with additional * parameters from the cluster configuration, but giving precedence to application-defined * parameters over configuration-defined parameters. */ @Test public void testLoadFileSystemStateBackendMixed() throws Exception { final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString(); final String checkpointDir = new Path(tmp.newFolder().toURI()).toString(); final String savepointDir = new Path(tmp.newFolder().toURI()).toString(); final Path expectedCheckpointsPath = new Path(new URI(appCheckpointDir)); final Path expectedSavepointsPath = new Path(savepointDir); final int threshold = 1000000; final int writeBufferSize = 4000000; final FsStateBackend backend = new FsStateBackend(new URI(appCheckpointDir), null, threshold, writeBufferSize, TernaryBoolean.TRUE); final Configuration config = new Configuration(); config.setString(backendKey, "jobmanager"); // this should not be picked up config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this should not be picked up config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir); config.set(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, MemorySize.parse("20")); // this should not be picked up config.setInteger(CheckpointingOptions.FS_WRITE_BUFFER_SIZE, 3000000); // this should not be picked up final StateBackend loadedBackend = StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null); assertTrue(loadedBackend instanceof FsStateBackend); final FsStateBackend fs = (FsStateBackend) loadedBackend; assertEquals(expectedCheckpointsPath, fs.getCheckpointPath()); assertEquals(expectedSavepointsPath, fs.getSavepointPath()); assertEquals(threshold, fs.getMinFileSizeThreshold()); assertEquals(writeBufferSize, fs.getWriteBufferSize()); }
Example #5
Source File: StateBackendLoadingTest.java From flink with Apache License 2.0 | 5 votes |
/** * Validates taking the application-defined file system state backend and adding with additional * parameters from the cluster configuration, but giving precedence to application-defined * parameters over configuration-defined parameters. */ @Test public void testLoadFileSystemStateBackendMixed() throws Exception { final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString(); final String checkpointDir = new Path(tmp.newFolder().toURI()).toString(); final String savepointDir = new Path(tmp.newFolder().toURI()).toString(); final Path expectedCheckpointsPath = new Path(new URI(appCheckpointDir)); final Path expectedSavepointsPath = new Path(savepointDir); final int threshold = 1000000; final int writeBufferSize = 4000000; final FsStateBackend backend = new FsStateBackend(new URI(appCheckpointDir), null, threshold, writeBufferSize, TernaryBoolean.TRUE); final Configuration config = new Configuration(); config.setString(backendKey, "jobmanager"); // this should not be picked up config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this should not be picked up config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir); config.setInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, 20); // this should not be picked up config.setInteger(CheckpointingOptions.FS_WRITE_BUFFER_SIZE, 3000000); // this should not be picked up final StateBackend loadedBackend = StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null); assertTrue(loadedBackend instanceof FsStateBackend); final FsStateBackend fs = (FsStateBackend) loadedBackend; assertEquals(expectedCheckpointsPath, fs.getCheckpointPath()); assertEquals(expectedSavepointsPath, fs.getSavepointPath()); assertEquals(threshold, fs.getMinFileSizeThreshold()); assertEquals(writeBufferSize, fs.getWriteBufferSize()); }
Example #6
Source File: RocksDBTtlStateTestBase.java From flink with Apache License 2.0 | 5 votes |
StateBackend createStateBackend(TernaryBoolean enableIncrementalCheckpointing) { String dbPath; String checkpointPath; try { dbPath = tempFolder.newFolder().getAbsolutePath(); checkpointPath = tempFolder.newFolder().toURI().toString(); } catch (IOException e) { throw new FlinkRuntimeException("Failed to init rocksdb test state backend"); } RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing); Configuration config = new Configuration(); backend = backend.configure(config, Thread.currentThread().getContextClassLoader()); backend.setDbStoragePath(dbPath); return backend; }
Example #7
Source File: RocksDBStateBackend.java From flink with Apache License 2.0 | 4 votes |
/** * @deprecated Use {@link #RocksDBStateBackend(StateBackend)} instead. */ @Deprecated public RocksDBStateBackend(AbstractStateBackend checkpointStreamBackend) { this(checkpointStreamBackend, TernaryBoolean.UNDEFINED); }
Example #8
Source File: IncSnapshotRocksDbTtlStateTest.java From flink with Apache License 2.0 | 4 votes |
@Override StateBackend createStateBackend() { return createStateBackend(TernaryBoolean.TRUE); }
Example #9
Source File: FullSnapshotRocksDbTtlStateTest.java From flink with Apache License 2.0 | 4 votes |
@Override StateBackend createStateBackend() { return createStateBackend(TernaryBoolean.FALSE); }
Example #10
Source File: RocksDBStateBackend.java From flink with Apache License 2.0 | 4 votes |
/** * @deprecated Use {@link #RocksDBStateBackend(StateBackend, TernaryBoolean)} instead. */ @Deprecated public RocksDBStateBackend(AbstractStateBackend checkpointStreamBackend, boolean enableIncrementalCheckpointing) { this(checkpointStreamBackend, TernaryBoolean.fromBoolean(enableIncrementalCheckpointing)); }
Example #11
Source File: RocksDBStateBackend.java From flink with Apache License 2.0 | 4 votes |
/** * @deprecated Use {@link #RocksDBStateBackend(StateBackend)} instead. */ @Deprecated public RocksDBStateBackend(AbstractStateBackend checkpointStreamBackend) { this(checkpointStreamBackend, TernaryBoolean.UNDEFINED); }
Example #12
Source File: FsStateBackend.java From flink with Apache License 2.0 | 4 votes |
/** * Creates a new state backend that stores its checkpoint data in the file system and location * defined by the given URI. * * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://') * must be accessible via {@link FileSystem#get(URI)}. * * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority * (host and port), or that the Hadoop configuration that describes that information must be in the * classpath. * * @param checkpointDirectory The path to write checkpoint metadata to. * @param defaultSavepointDirectory The path to write savepoints to. If null, the value from * the runtime configuration will be used, or savepoint * target locations need to be passed when triggering a savepoint. * @param fileStateSizeThreshold State below this size will be stored as part of the metadata, * rather than in files. If -1, the value configured in the * runtime configuration will be used, or the default value (1KB) * if nothing is configured. * @param writeBufferSize Write buffer size used to serialize state. If -1, the value configured in the * runtime configuration will be used, or the default value (4KB) * if nothing is configured. * @param asynchronousSnapshots Flag to switch between synchronous and asynchronous * snapshot mode. If UNDEFINED, the value configured in the * runtime configuration will be used. */ public FsStateBackend( URI checkpointDirectory, @Nullable URI defaultSavepointDirectory, int fileStateSizeThreshold, int writeBufferSize, TernaryBoolean asynchronousSnapshots) { super(checkNotNull(checkpointDirectory, "checkpoint directory is null"), defaultSavepointDirectory); checkNotNull(asynchronousSnapshots, "asynchronousSnapshots"); checkArgument(fileStateSizeThreshold >= -1 && fileStateSizeThreshold <= MAX_FILE_STATE_THRESHOLD, "The threshold for file state size must be in [-1, %s], where '-1' means to use " + "the value from the deployment's configuration.", MAX_FILE_STATE_THRESHOLD); checkArgument(writeBufferSize >= -1 , "The write buffer size must be not less than '-1', where '-1' means to use " + "the value from the deployment's configuration."); this.fileStateThreshold = fileStateSizeThreshold; this.writeBufferSize = writeBufferSize; this.asynchronousSnapshots = asynchronousSnapshots; }
Example #13
Source File: CsvTableSinkFactoryTest.java From flink with Apache License 2.0 | 4 votes |
@Parameterized.Parameters(name = "deriveSchema = {0}") public static TernaryBoolean[] getDeriveSchema() { return new TernaryBoolean[]{TernaryBoolean.TRUE, TernaryBoolean.FALSE, TernaryBoolean.UNDEFINED}; }
Example #14
Source File: RocksDBStateBackend.java From flink with Apache License 2.0 | 4 votes |
/** * @deprecated Use {@link #RocksDBStateBackend(StateBackend, TernaryBoolean)} instead. */ @Deprecated public RocksDBStateBackend(AbstractStateBackend checkpointStreamBackend, boolean enableIncrementalCheckpointing) { this(checkpointStreamBackend, TernaryBoolean.fromBoolean(enableIncrementalCheckpointing)); }
Example #15
Source File: FsStateBackend.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
/** * Creates a new state backend that stores its checkpoint data in the file system and location * defined by the given URI. * * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://') * must be accessible via {@link FileSystem#get(URI)}. * * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority * (host and port), or that the Hadoop configuration that describes that information must be in the * classpath. * * @param checkpointDirectory The path to write checkpoint metadata to. * @param defaultSavepointDirectory The path to write savepoints to. If null, the value from * the runtime configuration will be used, or savepoint * target locations need to be passed when triggering a savepoint. * @param fileStateSizeThreshold State below this size will be stored as part of the metadata, * rather than in files. If -1, the value configured in the * runtime configuration will be used, or the default value (1KB) * if nothing is configured. * @param asynchronousSnapshots Flag to switch between synchronous and asynchronous * snapshot mode. If UNDEFINED, the value configured in the * runtime configuration will be used. */ public FsStateBackend( URI checkpointDirectory, @Nullable URI defaultSavepointDirectory, int fileStateSizeThreshold, TernaryBoolean asynchronousSnapshots) { super(checkNotNull(checkpointDirectory, "checkpoint directory is null"), defaultSavepointDirectory); checkNotNull(asynchronousSnapshots, "asynchronousSnapshots"); checkArgument(fileStateSizeThreshold >= -1 && fileStateSizeThreshold <= MAX_FILE_STATE_THRESHOLD, "The threshold for file state size must be in [-1, %s], where '-1' means to use " + "the value from the deployment's configuration.", MAX_FILE_STATE_THRESHOLD); this.fileStateThreshold = fileStateSizeThreshold; this.asynchronousSnapshots = asynchronousSnapshots; }
Example #16
Source File: FullSnapshotRocksDbTtlStateTest.java From flink with Apache License 2.0 | 4 votes |
@Override StateBackend createStateBackend() { return createStateBackend(TernaryBoolean.FALSE); }
Example #17
Source File: IncSnapshotRocksDbTtlStateTest.java From flink with Apache License 2.0 | 4 votes |
@Override StateBackend createStateBackend() { return createStateBackend(TernaryBoolean.TRUE); }
Example #18
Source File: FsStateBackend.java From flink with Apache License 2.0 | 4 votes |
/** * Creates a new state backend that stores its checkpoint data in the file system and location * defined by the given URI. * * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://') * must be accessible via {@link FileSystem#get(URI)}. * * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority * (host and port), or that the Hadoop configuration that describes that information must be in the * classpath. * * @param checkpointDirectory The path to write checkpoint metadata to. * @param defaultSavepointDirectory The path to write savepoints to. If null, the value from * the runtime configuration will be used, or savepoint * target locations need to be passed when triggering a savepoint. * @param fileStateSizeThreshold State below this size will be stored as part of the metadata, * rather than in files. If -1, the value configured in the * runtime configuration will be used, or the default value (1KB) * if nothing is configured. * @param writeBufferSize Write buffer size used to serialize state. If -1, the value configured in the * runtime configuration will be used, or the default value (4KB) * if nothing is configured. * @param asynchronousSnapshots Flag to switch between synchronous and asynchronous * snapshot mode. If UNDEFINED, the value configured in the * runtime configuration will be used. */ public FsStateBackend( URI checkpointDirectory, @Nullable URI defaultSavepointDirectory, int fileStateSizeThreshold, int writeBufferSize, TernaryBoolean asynchronousSnapshots) { super(checkNotNull(checkpointDirectory, "checkpoint directory is null"), defaultSavepointDirectory); checkNotNull(asynchronousSnapshots, "asynchronousSnapshots"); checkArgument(fileStateSizeThreshold >= -1 && fileStateSizeThreshold <= MAX_FILE_STATE_THRESHOLD, "The threshold for file state size must be in [-1, %s], where '-1' means to use " + "the value from the deployment's configuration.", MAX_FILE_STATE_THRESHOLD); checkArgument(writeBufferSize >= -1 , "The write buffer size must be not less than '-1', where '-1' means to use " + "the value from the deployment's configuration."); this.fileStateThreshold = fileStateSizeThreshold; this.writeBufferSize = writeBufferSize; this.asynchronousSnapshots = asynchronousSnapshots; }
Example #19
Source File: IncSnapshotRocksDbTtlStateTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Override StateBackend createStateBackend() { return createStateBackend(TernaryBoolean.TRUE); }
Example #20
Source File: FullSnapshotRocksDbTtlStateTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Override StateBackend createStateBackend() { return createStateBackend(TernaryBoolean.FALSE); }
Example #21
Source File: RocksDBStateBackend.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
/** * @deprecated Use {@link #RocksDBStateBackend(StateBackend, TernaryBoolean)} instead. */ @Deprecated public RocksDBStateBackend(AbstractStateBackend checkpointStreamBackend, boolean enableIncrementalCheckpointing) { this(checkpointStreamBackend, TernaryBoolean.fromBoolean(enableIncrementalCheckpointing)); }
Example #22
Source File: RocksDBStateBackend.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
/** * @deprecated Use {@link #RocksDBStateBackend(StateBackend)} instead. */ @Deprecated public RocksDBStateBackend(AbstractStateBackend checkpointStreamBackend) { this(checkpointStreamBackend, TernaryBoolean.UNDEFINED); }
Example #23
Source File: RocksDBStateBackend.java From Flink-CEPplus with Apache License 2.0 | 3 votes |
/** * Creates a new {@code RocksDBStateBackend} that uses the given state backend to store its * checkpoint data streams. Typically, one would supply a filesystem or database state backend * here where the snapshots from RocksDB would be stored. * * <p>The snapshots of the RocksDB state will be stored using the given backend's * {@link StateBackend#createCheckpointStorage(JobID)}. * * @param checkpointStreamBackend The backend write the checkpoint streams to. * @param enableIncrementalCheckpointing True if incremental checkpointing is enabled. */ public RocksDBStateBackend(StateBackend checkpointStreamBackend, TernaryBoolean enableIncrementalCheckpointing) { this.checkpointStreamBackend = checkNotNull(checkpointStreamBackend); this.enableIncrementalCheckpointing = enableIncrementalCheckpointing; this.numberOfTransferingThreads = UNDEFINED_NUMBER_OF_TRANSFERING_THREADS; // for now, we use still the heap-based implementation as default this.priorityQueueStateType = PriorityQueueStateType.HEAP; this.defaultMetricOptions = new RocksDBNativeMetricOptions(); this.enableTtlCompactionFilter = TernaryBoolean.UNDEFINED; }
Example #24
Source File: RocksDBStateBackend.java From flink with Apache License 2.0 | 3 votes |
/** * Creates a new {@code RocksDBStateBackend} that uses the given state backend to store its * checkpoint data streams. Typically, one would supply a filesystem or database state backend * here where the snapshots from RocksDB would be stored. * * <p>The snapshots of the RocksDB state will be stored using the given backend's * {@link StateBackend#createCheckpointStorage(JobID)}. * * @param checkpointStreamBackend The backend write the checkpoint streams to. * @param enableIncrementalCheckpointing True if incremental checkpointing is enabled. */ public RocksDBStateBackend(StateBackend checkpointStreamBackend, TernaryBoolean enableIncrementalCheckpointing) { this.checkpointStreamBackend = checkNotNull(checkpointStreamBackend); this.enableIncrementalCheckpointing = enableIncrementalCheckpointing; this.numberOfTransferThreads = UNDEFINED_NUMBER_OF_TRANSFER_THREADS; this.defaultMetricOptions = new RocksDBNativeMetricOptions(); this.memoryConfiguration = new RocksDBMemoryConfiguration(); this.writeBatchSize = UNDEFINED_WRITE_BATCH_SIZE; }
Example #25
Source File: FsStateBackend.java From flink with Apache License 2.0 | 3 votes |
/** * Creates a new state backend that stores its checkpoint data in the file system and location * defined by the given URI. * * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://') * must be accessible via {@link FileSystem#get(URI)}. * * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority * (host and port), or that the Hadoop configuration that describes that information must be in the * classpath. * * @param checkpointDataUri The URI describing the filesystem (scheme and optionally authority), * and the path to the checkpoint data directory. * @param fileStateSizeThreshold State up to this size will be stored as part of the metadata, * rather than in files (-1 for default value). * @param asynchronousSnapshots Switch to enable asynchronous snapshots. */ public FsStateBackend( URI checkpointDataUri, int fileStateSizeThreshold, boolean asynchronousSnapshots) { this(checkpointDataUri, null, fileStateSizeThreshold, -1, TernaryBoolean.fromBoolean(asynchronousSnapshots)); }
Example #26
Source File: MemoryStateBackend.java From flink with Apache License 2.0 | 3 votes |
/** * Creates a new MemoryStateBackend, setting optionally the paths to persist checkpoint metadata * and savepoints to, as well as configuring state thresholds and asynchronous operations. * * <p><b>WARNING:</b> Increasing the size of this value beyond the default value * ({@value #DEFAULT_MAX_STATE_SIZE}) should be done with care. * The checkpointed state needs to be send to the JobManager via limited size RPC messages, and there * and the JobManager needs to be able to hold all aggregated state in its memory. * * @param checkpointPath The path to write checkpoint metadata to. If null, the value from * the runtime configuration will be used. * @param savepointPath The path to write savepoints to. If null, the value from * the runtime configuration will be used. * @param maxStateSize The maximal size of the serialized state. * @param asynchronousSnapshots Flag to switch between synchronous and asynchronous * snapshot mode. If null, the value configured in the * runtime configuration will be used. */ public MemoryStateBackend( @Nullable String checkpointPath, @Nullable String savepointPath, int maxStateSize, TernaryBoolean asynchronousSnapshots) { super(checkpointPath == null ? null : new Path(checkpointPath), savepointPath == null ? null : new Path(savepointPath)); checkArgument(maxStateSize > 0, "maxStateSize must be > 0"); this.maxStateSize = maxStateSize; this.asynchronousSnapshots = asynchronousSnapshots; }
Example #27
Source File: FsStateBackend.java From flink with Apache License 2.0 | 3 votes |
/** * Creates a new state backend that stores its checkpoint data in the file system and location * defined by the given URI. * * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://') * must be accessible via {@link FileSystem#get(URI)}. * * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority * (host and port), or that the Hadoop configuration that describes that information must be in the * classpath. * * @param checkpointDataUri The URI describing the filesystem (scheme and optionally authority), * and the path to the checkpoint data directory. * @param fileStateSizeThreshold State up to this size will be stored as part of the metadata, * rather than in files (-1 for default value). * @param asynchronousSnapshots Switch to enable asynchronous snapshots. */ public FsStateBackend( URI checkpointDataUri, int fileStateSizeThreshold, boolean asynchronousSnapshots) { this(checkpointDataUri, null, fileStateSizeThreshold, -1, TernaryBoolean.fromBoolean(asynchronousSnapshots)); }
Example #28
Source File: MemoryStateBackend.java From flink with Apache License 2.0 | 3 votes |
/** * Creates a new MemoryStateBackend, setting optionally the paths to persist checkpoint metadata * and savepoints to, as well as configuring state thresholds and asynchronous operations. * * <p><b>WARNING:</b> Increasing the size of this value beyond the default value * ({@value #DEFAULT_MAX_STATE_SIZE}) should be done with care. * The checkpointed state needs to be send to the JobManager via limited size RPC messages, and there * and the JobManager needs to be able to hold all aggregated state in its memory. * * @param checkpointPath The path to write checkpoint metadata to. If null, the value from * the runtime configuration will be used. * @param savepointPath The path to write savepoints to. If null, the value from * the runtime configuration will be used. * @param maxStateSize The maximal size of the serialized state. * @param asynchronousSnapshots Flag to switch between synchronous and asynchronous * snapshot mode. If null, the value configured in the * runtime configuration will be used. */ public MemoryStateBackend( @Nullable String checkpointPath, @Nullable String savepointPath, int maxStateSize, TernaryBoolean asynchronousSnapshots) { super(checkpointPath == null ? null : new Path(checkpointPath), savepointPath == null ? null : new Path(savepointPath)); checkArgument(maxStateSize > 0, "maxStateSize must be > 0"); this.maxStateSize = maxStateSize; this.asynchronousSnapshots = asynchronousSnapshots; }
Example #29
Source File: FsStateBackend.java From Flink-CEPplus with Apache License 2.0 | 3 votes |
/** * Creates a new state backend that stores its checkpoint data in the file system and location * defined by the given URI. * * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://') * must be accessible via {@link FileSystem#get(URI)}. * * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority * (host and port), or that the Hadoop configuration that describes that information must be in the * classpath. * * @param checkpointDataUri The URI describing the filesystem (scheme and optionally authority), * and the path to the checkpoint data directory. * @param fileStateSizeThreshold State up to this size will be stored as part of the metadata, * rather than in files (-1 for default value). * @param asynchronousSnapshots Switch to enable asynchronous snapshots. */ public FsStateBackend( URI checkpointDataUri, int fileStateSizeThreshold, boolean asynchronousSnapshots) { this(checkpointDataUri, null, fileStateSizeThreshold, TernaryBoolean.fromBoolean(asynchronousSnapshots)); }
Example #30
Source File: MemoryStateBackend.java From Flink-CEPplus with Apache License 2.0 | 3 votes |
/** * Creates a new MemoryStateBackend, setting optionally the paths to persist checkpoint metadata * and savepoints to, as well as configuring state thresholds and asynchronous operations. * * <p><b>WARNING:</b> Increasing the size of this value beyond the default value * ({@value #DEFAULT_MAX_STATE_SIZE}) should be done with care. * The checkpointed state needs to be send to the JobManager via limited size RPC messages, and there * and the JobManager needs to be able to hold all aggregated state in its memory. * * @param checkpointPath The path to write checkpoint metadata to. If null, the value from * the runtime configuration will be used. * @param savepointPath The path to write savepoints to. If null, the value from * the runtime configuration will be used. * @param maxStateSize The maximal size of the serialized state. * @param asynchronousSnapshots Flag to switch between synchronous and asynchronous * snapshot mode. If null, the value configured in the * runtime configuration will be used. */ public MemoryStateBackend( @Nullable String checkpointPath, @Nullable String savepointPath, int maxStateSize, TernaryBoolean asynchronousSnapshots) { super(checkpointPath == null ? null : new Path(checkpointPath), savepointPath == null ? null : new Path(savepointPath)); checkArgument(maxStateSize > 0, "maxStateSize must be > 0"); this.maxStateSize = maxStateSize; this.asynchronousSnapshots = asynchronousSnapshots; }