org.apache.flink.runtime.zookeeper.RetrievableStateStorageHelper Java Examples

The following examples show how to use org.apache.flink.runtime.zookeeper.RetrievableStateStorageHelper. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ZooKeeperMesosServices.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public MesosWorkerStore createMesosWorkerStore(Configuration configuration, Executor executor) throws Exception {
	RetrievableStateStorageHelper<MesosWorkerStore.Worker> stateStorageHelper =
		ZooKeeperUtils.createFileSystemStateStorage(configuration, "mesosWorkerStore");

	ZooKeeperStateHandleStore<MesosWorkerStore.Worker> zooKeeperStateHandleStore = zooKeeperUtilityFactory.createZooKeeperStateHandleStore(
		"/workers",
		stateStorageHelper);

	ZooKeeperSharedValue frameworkId = zooKeeperUtilityFactory.createSharedValue("/frameworkId", new byte[0]);
	ZooKeeperSharedCount totalTaskCount = zooKeeperUtilityFactory.createSharedCount("/taskCount", 0);

	return new ZooKeeperMesosWorkerStore(
		zooKeeperStateHandleStore,
		frameworkId,
		totalTaskCount);
}
Example #2
Source File: ZooKeeperMesosServices.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public MesosWorkerStore createMesosWorkerStore(Configuration configuration, Executor executor) throws Exception {
	RetrievableStateStorageHelper<MesosWorkerStore.Worker> stateStorageHelper =
		ZooKeeperUtils.createFileSystemStateStorage(configuration, "mesosWorkerStore");

	ZooKeeperStateHandleStore<MesosWorkerStore.Worker> zooKeeperStateHandleStore = zooKeeperUtilityFactory.createZooKeeperStateHandleStore(
		"/workers",
		stateStorageHelper);

	ZooKeeperSharedValue frameworkId = zooKeeperUtilityFactory.createSharedValue("/frameworkId", new byte[0]);
	ZooKeeperSharedCount totalTaskCount = zooKeeperUtilityFactory.createSharedCount("/taskCount", 0);

	return new ZooKeeperMesosWorkerStore(
		zooKeeperStateHandleStore,
		frameworkId,
		totalTaskCount);
}
Example #3
Source File: ZooKeeperMesosServices.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public MesosWorkerStore createMesosWorkerStore(Configuration configuration, Executor executor) throws Exception {
	RetrievableStateStorageHelper<MesosWorkerStore.Worker> stateStorageHelper =
		ZooKeeperUtils.createFileSystemStateStorage(configuration, "mesosWorkerStore");

	ZooKeeperStateHandleStore<MesosWorkerStore.Worker> zooKeeperStateHandleStore = zooKeeperUtilityFactory.createZooKeeperStateHandleStore(
		"/workers",
		stateStorageHelper);

	ZooKeeperSharedValue frameworkId = zooKeeperUtilityFactory.createSharedValue("/frameworkId", new byte[0]);
	ZooKeeperSharedCount totalTaskCount = zooKeeperUtilityFactory.createSharedCount("/taskCount", 0);

	return new ZooKeeperMesosWorkerStore(
		zooKeeperStateHandleStore,
		frameworkId,
		totalTaskCount);
}
Example #4
Source File: ZooKeeperUtils.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a {@link ZooKeeperSubmittedJobGraphStore} instance.
 *
 * @param client        The {@link CuratorFramework} ZooKeeper client to use
 * @param configuration {@link Configuration} object
 * @return {@link ZooKeeperSubmittedJobGraphStore} instance
 * @throws Exception if the submitted job graph store cannot be created
 */
public static ZooKeeperSubmittedJobGraphStore createSubmittedJobGraphs(
		CuratorFramework client,
		Configuration configuration) throws Exception {

	checkNotNull(configuration, "Configuration");

	RetrievableStateStorageHelper<SubmittedJobGraph> stateStorage = createFileSystemStateStorage(configuration, "submittedJobGraph");

	// ZooKeeper submitted jobs root dir
	String zooKeeperSubmittedJobsPath = configuration.getString(HighAvailabilityOptions.HA_ZOOKEEPER_JOBGRAPHS_PATH);

	// Ensure that the job graphs path exists
	client.newNamespaceAwareEnsurePath(zooKeeperSubmittedJobsPath)
		.ensure(client.getZookeeperClient());

	// All operations will have the path as root
	CuratorFramework facade = client.usingNamespace(client.getNamespace() + zooKeeperSubmittedJobsPath);

	final String zooKeeperFullSubmittedJobsPath = client.getNamespace() + zooKeeperSubmittedJobsPath;

	final ZooKeeperStateHandleStore<SubmittedJobGraph> zooKeeperStateHandleStore = new ZooKeeperStateHandleStore<>(facade, stateStorage);

	final PathChildrenCache pathCache = new PathChildrenCache(facade, "/", false);

	return new ZooKeeperSubmittedJobGraphStore(
		zooKeeperFullSubmittedJobsPath,
		zooKeeperStateHandleStore,
		pathCache);
}
Example #5
Source File: ZooKeeperUtils.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a {@link ZooKeeperCompletedCheckpointStore} instance.
 *
 * @param client                         The {@link CuratorFramework} ZooKeeper client to use
 * @param configuration                  {@link Configuration} object
 * @param jobId                          ID of job to create the instance for
 * @param maxNumberOfCheckpointsToRetain The maximum number of checkpoints to retain
 * @param executor to run ZooKeeper callbacks
 * @return {@link ZooKeeperCompletedCheckpointStore} instance
 * @throws Exception if the completed checkpoint store cannot be created
 */
public static CompletedCheckpointStore createCompletedCheckpoints(
		CuratorFramework client,
		Configuration configuration,
		JobID jobId,
		int maxNumberOfCheckpointsToRetain,
		Executor executor) throws Exception {

	checkNotNull(configuration, "Configuration");

	String checkpointsPath = configuration.getString(
		HighAvailabilityOptions.HA_ZOOKEEPER_CHECKPOINTS_PATH);

	RetrievableStateStorageHelper<CompletedCheckpoint> stateStorage = createFileSystemStateStorage(
		configuration,
		"completedCheckpoint");

	checkpointsPath += ZooKeeperSubmittedJobGraphStore.getPathForJob(jobId);

	final ZooKeeperCompletedCheckpointStore zooKeeperCompletedCheckpointStore = new ZooKeeperCompletedCheckpointStore(
		maxNumberOfCheckpointsToRetain,
		createZooKeeperStateHandleStore(client, checkpointsPath, stateStorage),
		executor);

	LOG.info("Initialized {} in '{}'.", ZooKeeperCompletedCheckpointStore.class.getSimpleName(), checkpointsPath);
	return zooKeeperCompletedCheckpointStore;
}
Example #6
Source File: ZooKeeperUtils.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a {@link ZooKeeperSubmittedJobGraphStore} instance.
 *
 * @param client        The {@link CuratorFramework} ZooKeeper client to use
 * @param configuration {@link Configuration} object
 * @return {@link ZooKeeperSubmittedJobGraphStore} instance
 * @throws Exception if the submitted job graph store cannot be created
 */
public static ZooKeeperSubmittedJobGraphStore createSubmittedJobGraphs(
		CuratorFramework client,
		Configuration configuration) throws Exception {

	checkNotNull(configuration, "Configuration");

	RetrievableStateStorageHelper<SubmittedJobGraph> stateStorage = createFileSystemStateStorage(configuration, "submittedJobGraph");

	// ZooKeeper submitted jobs root dir
	String zooKeeperSubmittedJobsPath = configuration.getString(HighAvailabilityOptions.HA_ZOOKEEPER_JOBGRAPHS_PATH);

	// Ensure that the job graphs path exists
	client.newNamespaceAwareEnsurePath(zooKeeperSubmittedJobsPath)
		.ensure(client.getZookeeperClient());

	// All operations will have the path as root
	CuratorFramework facade = client.usingNamespace(client.getNamespace() + zooKeeperSubmittedJobsPath);

	final String zooKeeperFullSubmittedJobsPath = client.getNamespace() + zooKeeperSubmittedJobsPath;

	final ZooKeeperStateHandleStore<SubmittedJobGraph> zooKeeperStateHandleStore = new ZooKeeperStateHandleStore<>(facade, stateStorage);

	final PathChildrenCache pathCache = new PathChildrenCache(facade, "/", false);

	return new ZooKeeperSubmittedJobGraphStore(
		zooKeeperFullSubmittedJobsPath,
		zooKeeperStateHandleStore,
		pathCache);
}
Example #7
Source File: ZooKeeperUtils.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a {@link ZooKeeperCompletedCheckpointStore} instance.
 *
 * @param client                         The {@link CuratorFramework} ZooKeeper client to use
 * @param configuration                  {@link Configuration} object
 * @param jobId                          ID of job to create the instance for
 * @param maxNumberOfCheckpointsToRetain The maximum number of checkpoints to retain
 * @param executor to run ZooKeeper callbacks
 * @return {@link ZooKeeperCompletedCheckpointStore} instance
 * @throws Exception if the completed checkpoint store cannot be created
 */
public static CompletedCheckpointStore createCompletedCheckpoints(
		CuratorFramework client,
		Configuration configuration,
		JobID jobId,
		int maxNumberOfCheckpointsToRetain,
		Executor executor) throws Exception {

	checkNotNull(configuration, "Configuration");

	String checkpointsPath = configuration.getString(
		HighAvailabilityOptions.HA_ZOOKEEPER_CHECKPOINTS_PATH);

	RetrievableStateStorageHelper<CompletedCheckpoint> stateStorage = createFileSystemStateStorage(
		configuration,
		"completedCheckpoint");

	checkpointsPath += ZooKeeperSubmittedJobGraphStore.getPathForJob(jobId);

	final ZooKeeperCompletedCheckpointStore zooKeeperCompletedCheckpointStore = new ZooKeeperCompletedCheckpointStore(
		maxNumberOfCheckpointsToRetain,
		createZooKeeperStateHandleStore(client, checkpointsPath, stateStorage),
		executor);

	LOG.info("Initialized {} in '{}'.", ZooKeeperCompletedCheckpointStore.class.getSimpleName(), checkpointsPath);
	return zooKeeperCompletedCheckpointStore;
}
Example #8
Source File: ZooKeeperUtils.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a {@link ZooKeeperJobGraphStore} instance.
 *
 * @param client        The {@link CuratorFramework} ZooKeeper client to use
 * @param configuration {@link Configuration} object
 * @return {@link ZooKeeperJobGraphStore} instance
 * @throws Exception if the submitted job graph store cannot be created
 */
public static ZooKeeperJobGraphStore createJobGraphs(
		CuratorFramework client,
		Configuration configuration) throws Exception {

	checkNotNull(configuration, "Configuration");

	RetrievableStateStorageHelper<JobGraph> stateStorage = createFileSystemStateStorage(
		configuration, HA_STORAGE_SUBMITTED_JOBGRAPH_PREFIX);

	// ZooKeeper submitted jobs root dir
	String zooKeeperJobsPath = configuration.getString(HighAvailabilityOptions.HA_ZOOKEEPER_JOBGRAPHS_PATH);

	// Ensure that the job graphs path exists
	client.newNamespaceAwareEnsurePath(zooKeeperJobsPath)
		.ensure(client.getZookeeperClient());

	// All operations will have the path as root
	CuratorFramework facade = client.usingNamespace(client.getNamespace() + zooKeeperJobsPath);

	final String zooKeeperFullJobsPath = client.getNamespace() + zooKeeperJobsPath;

	final ZooKeeperStateHandleStore<JobGraph> zooKeeperStateHandleStore = new ZooKeeperStateHandleStore<>(facade, stateStorage);

	final PathChildrenCache pathCache = new PathChildrenCache(facade, "/", false);

	return new ZooKeeperJobGraphStore(
		zooKeeperFullJobsPath,
		zooKeeperStateHandleStore,
		pathCache);
}
Example #9
Source File: ZooKeeperUtils.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a {@link ZooKeeperCompletedCheckpointStore} instance.
 *
 * @param client                         The {@link CuratorFramework} ZooKeeper client to use
 * @param configuration                  {@link Configuration} object
 * @param jobId                          ID of job to create the instance for
 * @param maxNumberOfCheckpointsToRetain The maximum number of checkpoints to retain
 * @param executor to run ZooKeeper callbacks
 * @return {@link ZooKeeperCompletedCheckpointStore} instance
 * @throws Exception if the completed checkpoint store cannot be created
 */
public static CompletedCheckpointStore createCompletedCheckpoints(
		CuratorFramework client,
		Configuration configuration,
		JobID jobId,
		int maxNumberOfCheckpointsToRetain,
		Executor executor) throws Exception {

	checkNotNull(configuration, "Configuration");

	String checkpointsPath = configuration.getString(
		HighAvailabilityOptions.HA_ZOOKEEPER_CHECKPOINTS_PATH);

	RetrievableStateStorageHelper<CompletedCheckpoint> stateStorage = createFileSystemStateStorage(
		configuration,
		HA_STORAGE_COMPLETED_CHECKPOINT);

	checkpointsPath += ZooKeeperJobGraphStore.getPathForJob(jobId);

	final ZooKeeperCompletedCheckpointStore zooKeeperCompletedCheckpointStore = new ZooKeeperCompletedCheckpointStore(
		maxNumberOfCheckpointsToRetain,
		createZooKeeperStateHandleStore(client, checkpointsPath, stateStorage),
		executor);

	LOG.info("Initialized {} in '{}'.", ZooKeeperCompletedCheckpointStore.class.getSimpleName(), checkpointsPath);
	return zooKeeperCompletedCheckpointStore;
}
Example #10
Source File: ZooKeeperUtils.java    From Flink-CEPplus with Apache License 2.0 3 votes vote down vote up 
/**
 * Creates an instance of {@link ZooKeeperStateHandleStore}.
 *
 * @param client       ZK client
 * @param path         Path to use for the client namespace
 * @param stateStorage RetrievableStateStorageHelper that persist the actual state and whose
 *                     returned state handle is then written to ZooKeeper
 * @param <T>          Type of state
 * @return {@link ZooKeeperStateHandleStore} instance
 * @throws Exception ZK errors
 */
public static <T extends Serializable> ZooKeeperStateHandleStore<T> createZooKeeperStateHandleStore(
		final CuratorFramework client,
		final String path,
		final RetrievableStateStorageHelper<T> stateStorage) throws Exception {
	return new ZooKeeperStateHandleStore<>(useNamespaceAndEnsurePath(client, path), stateStorage);
}
Example #11
Source File: ZooKeeperUtils.java    From flink with Apache License 2.0 3 votes vote down vote up 
/**
 * Creates an instance of {@link ZooKeeperStateHandleStore}.
 *
 * @param client       ZK client
 * @param path         Path to use for the client namespace
 * @param stateStorage RetrievableStateStorageHelper that persist the actual state and whose
 *                     returned state handle is then written to ZooKeeper
 * @param <T>          Type of state
 * @return {@link ZooKeeperStateHandleStore} instance
 * @throws Exception ZK errors
 */
public static <T extends Serializable> ZooKeeperStateHandleStore<T> createZooKeeperStateHandleStore(
		final CuratorFramework client,
		final String path,
		final RetrievableStateStorageHelper<T> stateStorage) throws Exception {
	return new ZooKeeperStateHandleStore<>(useNamespaceAndEnsurePath(client, path), stateStorage);
}
Example #12
Source File: ZooKeeperUtils.java    From flink with Apache License 2.0 3 votes vote down vote up 
/**
 * Creates an instance of {@link ZooKeeperStateHandleStore}.
 *
 * @param client       ZK client
 * @param path         Path to use for the client namespace
 * @param stateStorage RetrievableStateStorageHelper that persist the actual state and whose
 *                     returned state handle is then written to ZooKeeper
 * @param <T>          Type of state
 * @return {@link ZooKeeperStateHandleStore} instance
 * @throws Exception ZK errors
 */
public static <T extends Serializable> ZooKeeperStateHandleStore<T> createZooKeeperStateHandleStore(
		final CuratorFramework client,
		final String path,
		final RetrievableStateStorageHelper<T> stateStorage) throws Exception {
	return new ZooKeeperStateHandleStore<>(useNamespaceAndEnsurePath(client, path), stateStorage);
}