Java Code Examples for org.apache.flink.api.common.cache.DistributedCache#DistributedCacheEntry

The following examples show how to use org.apache.flink.api.common.cache.DistributedCache#DistributedCacheEntry . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FileCacheReadsFromBlobTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testFileDownloadedFromBlob() throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID = new ExecutionAttemptID();

	final String fileName = "test_file";
	// copy / create the file
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		fileName,
		false,
		InstantiationUtil.serializeObject(permanentBlobKey));
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID);

	final Path dstPath = copyResult.get();
	final String actualContent = Files.toString(new File(dstPath.toUri()), StandardCharsets.UTF_8);
	assertTrue(dstPath.getFileSystem().exists(dstPath));
	assertEquals(testFileContent, actualContent);
}
Example 2
Source File: FileCacheReadsFromBlobTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testFileDownloadedFromBlob() throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID = new ExecutionAttemptID();

	final String fileName = "test_file";
	// copy / create the file
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		fileName,
		false,
		InstantiationUtil.serializeObject(permanentBlobKey));
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID);

	final Path dstPath = copyResult.get();
	final String actualContent = Files.toString(new File(dstPath.toUri()), StandardCharsets.UTF_8);
	assertTrue(dstPath.getFileSystem().exists(dstPath));
	assertEquals(testFileContent, actualContent);
}
Example 3
Source File: FileCacheDirectoriesTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testDirectoryDownloadedFromBlob() throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID = new ExecutionAttemptID();

	final String fileName = "test_file";
	// copy / create the file
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		fileName,
		false,
		InstantiationUtil.serializeObject(permanentBlobKey),
		true);
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID);

	final Path dstPath = copyResult.get();
	final FileSystem fs = dstPath.getFileSystem();
	final FileStatus fileStatus = fs.getFileStatus(dstPath);
	assertTrue(fileStatus.isDir());

	final Path cacheFile = new Path(dstPath, "cacheFile");
	assertTrue(fs.exists(cacheFile));
	final String actualContent = FileUtils.readFileUtf8(new File(cacheFile.getPath()));
	assertEquals(testFileContent, actualContent);
}
Example 4
Source File: PlanGenerator.java    From flink with Apache License 2.0 5 votes vote down vote up 
public PlanGenerator(
		List<DataSink<?>> sinks,
		ExecutionConfig config,
		int defaultParallelism,
		List<Tuple2<String, DistributedCache.DistributedCacheEntry>> cacheFile,
		String jobName) {
	this.sinks = checkNotNull(sinks);
	this.config = checkNotNull(config);
	this.cacheFile = checkNotNull(cacheFile);
	this.jobName = checkNotNull(jobName);
	this.defaultParallelism = defaultParallelism;
}
Example 5
Source File: FileCacheDirectoriesTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testDirectoryCleanUp() throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID1 = new ExecutionAttemptID();
	ExecutionAttemptID attemptID2 = new ExecutionAttemptID();

	final String fileName = "test_file";
	// copy / create the file
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		fileName,
		false,
		InstantiationUtil.serializeObject(permanentBlobKey),
		true);
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID1);
	fileCache.createTmpFile(fileName, entry, jobID, attemptID2);

	final Path dstPath = copyResult.get();
	final FileSystem fs = dstPath.getFileSystem();
	final FileStatus fileStatus = fs.getFileStatus(dstPath);
	final Path cacheFile = new Path(dstPath, "cacheFile");
	assertTrue(fileStatus.isDir());
	assertTrue(fs.exists(cacheFile));

	fileCache.releaseJob(jobID, attemptID1);
	// still should be available
	assertTrue(fileStatus.isDir());
	assertTrue(fs.exists(cacheFile));

	fileCache.releaseJob(jobID, attemptID2);
	// still should be available, file will be deleted after cleanupInterval
	assertTrue(fileStatus.isDir());
	assertTrue(fs.exists(cacheFile));

	// after a while, the file should disappear
	assertEquals(CLEANUP_INTERVAL, executorService.lastDelayMillis);
	executorService.lastDeleteProcess.run();

	assertFalse(fs.exists(dstPath));
	assertFalse(fs.exists(cacheFile));
}
Example 6
Source File: FileCacheDirectoriesTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testDirectoryDownloadedFromBlob() throws Exception {
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		"test_file",
		false,
		InstantiationUtil.serializeObject(permanentBlobKey),
		true);

	testDirectoryDownloaded(entry);
}
Example 7
Source File: JobGraph.java    From flink with Apache License 2.0 5 votes vote down vote up 
public void writeUserArtifactEntriesToConfiguration() {
	for (Map.Entry<String, DistributedCache.DistributedCacheEntry> userArtifact : userArtifacts.entrySet()) {
		DistributedCache.writeFileInfoToConfig(
			userArtifact.getKey(),
			userArtifact.getValue(),
			jobConfiguration
		);
	}
}
Example 8
Source File: JobGraphGeneratorTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private static void assertState(DistributedCache.DistributedCacheEntry entry, boolean isExecutable, boolean isZipped) throws IOException {
	assertNotNull(entry);
	assertEquals(isExecutable, entry.isExecutable);
	assertEquals(isZipped, entry.isZipped);
	org.apache.flink.core.fs.Path filePath = new org.apache.flink.core.fs.Path(entry.filePath);
	assertTrue(filePath.getFileSystem().exists(filePath));
	assertFalse(filePath.getFileSystem().getFileStatus(filePath).isDir());
}
Example 9
Source File: JobGraph.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Adds the path of a custom file required to run the job on a task manager.
 *
 * @param name a name under which this artifact will be accessible through {@link DistributedCache}
 * @param file path of a custom file required to run the job on a task manager
 */
public void addUserArtifact(String name, DistributedCache.DistributedCacheEntry file) {
	if (file == null) {
		throw new IllegalArgumentException();
	}

	userArtifacts.putIfAbsent(name, file);
}
Example 10
Source File: FileCacheDirectoriesTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testDirectoryCleanUp() throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID1 = new ExecutionAttemptID();
	ExecutionAttemptID attemptID2 = new ExecutionAttemptID();

	final String fileName = "test_file";
	// copy / create the file
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		fileName,
		false,
		InstantiationUtil.serializeObject(permanentBlobKey),
		true);
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID1);
	fileCache.createTmpFile(fileName, entry, jobID, attemptID2);

	final Path dstPath = copyResult.get();
	final FileSystem fs = dstPath.getFileSystem();
	final FileStatus fileStatus = fs.getFileStatus(dstPath);
	final Path cacheFile = new Path(dstPath, "cacheFile");
	assertTrue(fileStatus.isDir());
	assertTrue(fs.exists(cacheFile));

	fileCache.releaseJob(jobID, attemptID1);
	// still should be available
	assertTrue(fileStatus.isDir());
	assertTrue(fs.exists(cacheFile));

	fileCache.releaseJob(jobID, attemptID2);
	// still should be available, file will be deleted after cleanupInterval
	assertTrue(fileStatus.isDir());
	assertTrue(fs.exists(cacheFile));

	// after a while, the file should disappear
	assertEquals(CLEANUP_INTERVAL, executorService.lastDelayMillis);
	executorService.lastDeleteProcess.run();

	assertFalse(fs.exists(dstPath));
	assertFalse(fs.exists(cacheFile));
}
Example 11
Source File: JobGraphGeneratorTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Test
public void testArtifactCompression() throws IOException {
	Path plainFile1 = tmp.newFile("plainFile1").toPath();
	Path plainFile2 = tmp.newFile("plainFile2").toPath();

	Path directory1 = tmp.newFolder("directory1").toPath();
	Files.createDirectory(directory1.resolve("containedFile1"));

	Path directory2 = tmp.newFolder("directory2").toPath();
	Files.createDirectory(directory2.resolve("containedFile2"));

	JobGraph jb = new JobGraph();

	final String executableFileName = "executableFile";
	final String nonExecutableFileName = "nonExecutableFile";
	final String executableDirName = "executableDir";
	final String nonExecutableDirName = "nonExecutableDIr";

	Collection<Tuple2<String, DistributedCache.DistributedCacheEntry>> originalArtifacts = Arrays.asList(
		Tuple2.of(executableFileName, new DistributedCache.DistributedCacheEntry(plainFile1.toString(), true)),
		Tuple2.of(nonExecutableFileName, new DistributedCache.DistributedCacheEntry(plainFile2.toString(), false)),
		Tuple2.of(executableDirName, new DistributedCache.DistributedCacheEntry(directory1.toString(), true)),
		Tuple2.of(nonExecutableDirName, new DistributedCache.DistributedCacheEntry(directory2.toString(), false))
	);

	JobGraphGenerator.addUserArtifactEntries(originalArtifacts, jb);

	Map<String, DistributedCache.DistributedCacheEntry> submittedArtifacts = jb.getUserArtifacts();

	DistributedCache.DistributedCacheEntry executableFileEntry = submittedArtifacts.get(executableFileName);
	assertState(executableFileEntry, true, false);

	DistributedCache.DistributedCacheEntry nonExecutableFileEntry = submittedArtifacts.get(nonExecutableFileName);
	assertState(nonExecutableFileEntry, false, false);

	DistributedCache.DistributedCacheEntry executableDirEntry = submittedArtifacts.get(executableDirName);
	assertState(executableDirEntry, true, true);

	DistributedCache.DistributedCacheEntry nonExecutableDirEntry = submittedArtifacts.get(nonExecutableDirName);
	assertState(nonExecutableDirEntry, false, true);
}
Example 12
Source File: JobGraphGeneratorTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testArtifactCompression() throws IOException {
	Path plainFile1 = tmp.newFile("plainFile1").toPath();
	Path plainFile2 = tmp.newFile("plainFile2").toPath();

	Path directory1 = tmp.newFolder("directory1").toPath();
	Files.createDirectory(directory1.resolve("containedFile1"));

	Path directory2 = tmp.newFolder("directory2").toPath();
	Files.createDirectory(directory2.resolve("containedFile2"));

	JobGraph jb = new JobGraph();

	final String executableFileName = "executableFile";
	final String nonExecutableFileName = "nonExecutableFile";
	final String executableDirName = "executableDir";
	final String nonExecutableDirName = "nonExecutableDIr";

	Collection<Tuple2<String, DistributedCache.DistributedCacheEntry>> originalArtifacts = Arrays.asList(
		Tuple2.of(executableFileName, new DistributedCache.DistributedCacheEntry(plainFile1.toString(), true)),
		Tuple2.of(nonExecutableFileName, new DistributedCache.DistributedCacheEntry(plainFile2.toString(), false)),
		Tuple2.of(executableDirName, new DistributedCache.DistributedCacheEntry(directory1.toString(), true)),
		Tuple2.of(nonExecutableDirName, new DistributedCache.DistributedCacheEntry(directory2.toString(), false))
	);

	JobGraphUtils.addUserArtifactEntries(originalArtifacts, jb);

	Map<String, DistributedCache.DistributedCacheEntry> submittedArtifacts = jb.getUserArtifacts();

	DistributedCache.DistributedCacheEntry executableFileEntry = submittedArtifacts.get(executableFileName);
	assertState(executableFileEntry, true, false);

	DistributedCache.DistributedCacheEntry nonExecutableFileEntry = submittedArtifacts.get(nonExecutableFileName);
	assertState(nonExecutableFileEntry, false, false);

	DistributedCache.DistributedCacheEntry executableDirEntry = submittedArtifacts.get(executableDirName);
	assertState(executableDirEntry, true, true);

	DistributedCache.DistributedCacheEntry nonExecutableDirEntry = submittedArtifacts.get(nonExecutableDirName);
	assertState(nonExecutableDirEntry, false, true);
}
Example 13
Source File: StreamGraph.java    From flink with Apache License 2.0 4 votes vote down vote up 
public Collection<Tuple2<String, DistributedCache.DistributedCacheEntry>> getUserArtifacts() {
	return userArtifacts;
}
Example 14
Source File: CollectionExecutor.java    From flink with Apache License 2.0 4 votes vote down vote up 
private void initCache(Set<Map.Entry<String, DistributedCache.DistributedCacheEntry>> files){
	for(Map.Entry<String, DistributedCache.DistributedCacheEntry> file: files){
		Future<Path> doNothing = new CompletedFuture(new Path(file.getValue().filePath));
		cachedFiles.put(file.getKey(), doNothing);
	}
}
Example 15
Source File: StreamGraph.java    From flink with Apache License 2.0 4 votes vote down vote up 
public void setUserArtifacts(Collection<Tuple2<String, DistributedCache.DistributedCacheEntry>> userArtifacts) {
	this.userArtifacts = userArtifacts;
}
Example 16
Source File: JobGraphGeneratorTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testArtifactCompression() throws IOException {
	Path plainFile1 = tmp.newFile("plainFile1").toPath();
	Path plainFile2 = tmp.newFile("plainFile2").toPath();

	Path directory1 = tmp.newFolder("directory1").toPath();
	Files.createDirectory(directory1.resolve("containedFile1"));

	Path directory2 = tmp.newFolder("directory2").toPath();
	Files.createDirectory(directory2.resolve("containedFile2"));

	JobGraph jb = new JobGraph();

	final String executableFileName = "executableFile";
	final String nonExecutableFileName = "nonExecutableFile";
	final String executableDirName = "executableDir";
	final String nonExecutableDirName = "nonExecutableDIr";

	Collection<Tuple2<String, DistributedCache.DistributedCacheEntry>> originalArtifacts = Arrays.asList(
		Tuple2.of(executableFileName, new DistributedCache.DistributedCacheEntry(plainFile1.toString(), true)),
		Tuple2.of(nonExecutableFileName, new DistributedCache.DistributedCacheEntry(plainFile2.toString(), false)),
		Tuple2.of(executableDirName, new DistributedCache.DistributedCacheEntry(directory1.toString(), true)),
		Tuple2.of(nonExecutableDirName, new DistributedCache.DistributedCacheEntry(directory2.toString(), false))
	);

	JobGraphGenerator.addUserArtifactEntries(originalArtifacts, jb);

	Map<String, DistributedCache.DistributedCacheEntry> submittedArtifacts = jb.getUserArtifacts();

	DistributedCache.DistributedCacheEntry executableFileEntry = submittedArtifacts.get(executableFileName);
	assertState(executableFileEntry, true, false);

	DistributedCache.DistributedCacheEntry nonExecutableFileEntry = submittedArtifacts.get(nonExecutableFileName);
	assertState(nonExecutableFileEntry, false, false);

	DistributedCache.DistributedCacheEntry executableDirEntry = submittedArtifacts.get(executableDirName);
	assertState(executableDirEntry, true, true);

	DistributedCache.DistributedCacheEntry nonExecutableDirEntry = submittedArtifacts.get(nonExecutableDirName);
	assertState(nonExecutableDirEntry, false, true);
}
Example 17
Source File: CollectionExecutor.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
private void initCache(Set<Map.Entry<String, DistributedCache.DistributedCacheEntry>> files){
	for(Map.Entry<String, DistributedCache.DistributedCacheEntry> file: files){
		Future<Path> doNothing = new CompletedFuture(new Path(file.getValue().filePath));
		cachedFiles.put(file.getKey(), doNothing);
	}
}
Example 18
Source File: JobGraph.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Gets the list of assigned user jar paths.
 *
 * @return The list of assigned user jar paths
 */
public Map<String, DistributedCache.DistributedCacheEntry> getUserArtifacts() {
	return userArtifacts;
}
Example 19
Source File: JobGraph.java    From Flink-CEPplus with Apache License 2.0 2 votes vote down vote up 
/**
 * Gets the list of assigned user jar paths.
 *
 * @return The list of assigned user jar paths
 */
public Map<String, DistributedCache.DistributedCacheEntry> getUserArtifacts() {
	return userArtifacts;
}
Example 20
Source File: PlanGenerator.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Registers all files that were registered at this execution environment's cache registry of the
 * given plan's cache registry.
 *
 * @param p The plan to register files at.
 * @throws IOException Thrown if checks for existence and sanity fail.
 */
private void registerCachedFilesWithPlan(Plan p) throws IOException {
	for (Tuple2<String, DistributedCache.DistributedCacheEntry> entry : cacheFile) {
		p.registerCachedFile(entry.f0, entry.f1);
	}
}