Java Code Examples for org.apache.flink.runtime.blob.BlobUtils#createBlobStoreFromConfig()

The following examples show how to use org.apache.flink.runtime.blob.BlobUtils#createBlobStoreFromConfig() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HDFSTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed corrupted JARs are
 * recognised during the download via a {@link org.apache.flink.runtime.blob.BlobServer}.
 */
@Test
public void testBlobServerCorruptedFile() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobServerCorruptionTest.testGetFailsFromCorruptFile(config, blobStoreService, exception);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
Example 2
Source File: HighAvailabilityServicesUtils.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static HighAvailabilityServices createAvailableOrEmbeddedServices(
	Configuration config,
	Executor executor) throws Exception {
	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(config);

	switch (highAvailabilityMode) {
		case NONE:
			return new EmbeddedHaServices(executor);

		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(config),
				executor,
				config,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(config, executor);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
Example 3
Source File: HDFSTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed corrupted JARs are
 * recognised during the download via a BLOB cache.
 */
@Test
public void testBlobCacheCorruptedFile() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobCacheCorruptionTest
			.testGetFailsFromCorruptFile(new JobID(), config, blobStoreService, exception);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
Example 4
Source File: HDFSTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from any
 * participating BlobServer when uploaded via a BLOB cache.
 */
@Test
public void testBlobCacheRecovery() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobCacheRecoveryTest.testBlobCacheRecovery(config, blobStoreService);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
Example 5
Source File: HDFSTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed corrupted JARs are
 * recognised during the download via a {@link org.apache.flink.runtime.blob.BlobServer}.
 */
@Test
public void testBlobServerCorruptedFile() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobServerCorruptionTest.testGetFailsFromCorruptFile(config, blobStoreService, exception);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
Example 6
Source File: HDFSTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from any
 * participating BlobServer when talking to the {@link org.apache.flink.runtime.blob.BlobServer} directly.
 */
@Test
public void testBlobServerRecovery() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobServerRecoveryTest.testBlobServerRecovery(config, blobStoreService);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
Example 7
Source File: HighAvailabilityServicesUtils.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static HighAvailabilityServices createAvailableOrEmbeddedServices(
	Configuration config,
	Executor executor) throws Exception {
	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(config);

	switch (highAvailabilityMode) {
		case NONE:
			return new EmbeddedHaServices(executor);

		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(config),
				executor,
				config,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(config, executor);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
Example 8
Source File: HDFSTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed corrupted JARs are
 * recognised during the download via a BLOB cache.
 */
@Test
public void testBlobCacheCorruptedFile() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobCacheCorruptionTest
			.testGetFailsFromCorruptFile(new JobID(), config, blobStoreService, exception);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
Example 9
Source File: HDFSTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from any
 * participating BlobServer when uploaded via a BLOB cache.
 */
@Test
public void testBlobCacheRecovery() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobCacheRecoveryTest.testBlobCacheRecovery(config, blobStoreService);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
Example 10
Source File: HDFSTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed corrupted JARs are
 * recognised during the download via a {@link org.apache.flink.runtime.blob.BlobServer}.
 */
@Test
public void testBlobServerCorruptedFile() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobServerCorruptionTest.testGetFailsFromCorruptFile(config, blobStoreService, exception);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
Example 11
Source File: HDFSTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from any
 * participating BlobServer when talking to the {@link org.apache.flink.runtime.blob.BlobServer} directly.
 */
@Test
public void testBlobServerRecovery() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobServerRecoveryTest.testBlobServerRecovery(config, blobStoreService);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
Example 12
Source File: HighAvailabilityServicesUtils.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public static HighAvailabilityServices createAvailableOrEmbeddedServices(
	Configuration config,
	Executor executor) throws Exception {
	HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(config);

	switch (highAvailabilityMode) {
		case NONE:
			return new EmbeddedHaServices(executor);

		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(config),
				executor,
				config,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(config, executor);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
Example 13
Source File: HDFSTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed corrupted JARs are
 * recognised during the download via a BLOB cache.
 */
@Test
public void testBlobCacheCorruptedFile() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobCacheCorruptionTest
			.testGetFailsFromCorruptFile(new JobID(), config, blobStoreService, exception);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
Example 14
Source File: HDFSTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from any
 * participating BlobServer when uploaded via a BLOB cache.
 */
@Test
public void testBlobCacheRecovery() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobCacheRecoveryTest.testBlobCacheRecovery(config, blobStoreService);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
Example 15
Source File: HDFSTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from any
 * participating BlobServer when talking to the {@link org.apache.flink.runtime.blob.BlobServer} directly.
 */
@Test
public void testBlobServerRecovery() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobServerRecoveryTest.testBlobServerRecovery(config, blobStoreService);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
Example 16
Source File: ZooKeeperDefaultDispatcherRunnerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Before
public void setup() throws IOException {
	fatalErrorHandler = new TestingFatalErrorHandler();
	configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().getAbsolutePath());

	clusterHaStorageDir = new File(HighAvailabilityServicesUtils.getClusterHighAvailableStoragePath(configuration).toString());
	blobServer = new BlobServer(configuration, BlobUtils.createBlobStoreFromConfig(configuration));
}
Example 17
Source File: HighAvailabilityServicesUtils.java    From flink with Apache License 2.0 4 votes vote down vote up 
public static HighAvailabilityServices createHighAvailabilityServices(
	Configuration configuration,
	Executor executor,
	AddressResolution addressResolution) throws Exception {

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);

			final String jobManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				JobMaster.JOB_MANAGER_NAME,
				addressResolution,
				configuration);
			final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				ResourceManager.RESOURCE_MANAGER_NAME,
				addressResolution,
				configuration);
			final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				Dispatcher.DISPATCHER_NAME,
				addressResolution,
				configuration);

			final String address = checkNotNull(configuration.getString(RestOptions.ADDRESS),
				"%s must be set",
				RestOptions.ADDRESS.key());
			final int port = configuration.getInteger(RestOptions.PORT);
			final boolean enableSSL = SSLUtils.isRestSSLEnabled(configuration);
			final String protocol = enableSSL ? "https://" : "http://";

			return new StandaloneHaServices(
				resourceManagerRpcUrl,
				dispatcherRpcUrl,
				jobManagerRpcUrl,
				String.format("%s%s:%s", protocol, address, port));
		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(configuration),
				executor,
				configuration,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(configuration, executor);

		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}
Example 18
Source File: HighAvailabilityServicesUtils.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public static HighAvailabilityServices createHighAvailabilityServices(
	Configuration configuration,
	Executor executor,
	AddressResolution addressResolution) throws Exception {

	HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);

			final String jobManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				JobMaster.JOB_MANAGER_NAME,
				addressResolution,
				configuration);
			final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				ResourceManager.RESOURCE_MANAGER_NAME,
				addressResolution,
				configuration);
			final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				Dispatcher.DISPATCHER_NAME,
				addressResolution,
				configuration);

			final String address = checkNotNull(configuration.getString(RestOptions.ADDRESS),
				"%s must be set",
				RestOptions.ADDRESS.key());
			final int port = configuration.getInteger(RestOptions.PORT);
			final boolean enableSSL = SSLUtils.isRestSSLEnabled(configuration);
			final String protocol = enableSSL ? "https://" : "http://";

			return new StandaloneHaServices(
				resourceManagerRpcUrl,
				dispatcherRpcUrl,
				jobManagerRpcUrl,
				String.format("%s%s:%s", protocol, address, port));
		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(configuration),
				executor,
				configuration,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(configuration, executor);

		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}
Example 19
Source File: HighAvailabilityServicesUtils.java    From flink with Apache License 2.0 4 votes vote down vote up 
public static HighAvailabilityServices createHighAvailabilityServices(
	Configuration configuration,
	Executor executor,
	AddressResolution addressResolution) throws Exception {

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);

			final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				AkkaRpcServiceUtils.createWildcardName(ResourceManager.RESOURCE_MANAGER_NAME),
				addressResolution,
				configuration);
			final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				AkkaRpcServiceUtils.createWildcardName(Dispatcher.DISPATCHER_NAME),
				addressResolution,
				configuration);
			final String webMonitorAddress = getWebMonitorAddress(
				configuration,
				addressResolution);

			return new StandaloneHaServices(
				resourceManagerRpcUrl,
				dispatcherRpcUrl,
				webMonitorAddress);
		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(configuration),
				executor,
				configuration,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(configuration, executor);

		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}