org.apache.flink.util.FileUtils Java Examples

@Test
public void testDirectoryDownloadedFromBlob() throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID = new ExecutionAttemptID();

	final String fileName = "test_file";
	// copy / create the file
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		fileName,
		false,
		InstantiationUtil.serializeObject(permanentBlobKey),
		true);
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID);

	final Path dstPath = copyResult.get();
	final FileSystem fs = dstPath.getFileSystem();
	final FileStatus fileStatus = fs.getFileStatus(dstPath);
	assertTrue(fileStatus.isDir());

	final Path cacheFile = new Path(dstPath, "cacheFile");
	assertTrue(fs.exists(cacheFile));
	final String actualContent = FileUtils.readFileUtf8(new File(cacheFile.getPath()));
	assertEquals(testFileContent, actualContent);
}
 

private void testDirectoryDownloaded(DistributedCache.DistributedCacheEntry entry) throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID = new ExecutionAttemptID();

	// copy / create the file
	final String fileName = "test_file";
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID);

	final Path dstPath = copyResult.get();
	final FileSystem fs = dstPath.getFileSystem();
	final FileStatus fileStatus = fs.getFileStatus(dstPath);
	assertTrue(fileStatus.isDir());

	final Path cacheFile = new Path(dstPath, "cacheFile");
	assertTrue(fs.exists(cacheFile));
	final String actualContent = FileUtils.readFileUtf8(new File(cacheFile.getPath()));
	assertEquals(testFileContent, actualContent);
}
 

@Before
public void before() throws Exception{
	File resultFile = tempFolder.newFile();
	//Delete file because the Scala API does not respect WriteMode set by the configuration
	resultFile.delete();
	resultPath = resultFile.toURI().toString();

	File verticesFile = tempFolder.newFile();
	FileUtils.writeFileUtf8(verticesFile, PageRankData.VERTICES);

	File edgesFile = tempFolder.newFile();
	FileUtils.writeFileUtf8(edgesFile, PageRankData.EDGES);

	verticesPath = verticesFile.toURI().toString();
	edgesPath = edgesFile.toURI().toString();
}
 

/**
 * Tests that the {@link AbstractTaskManagerFileHandler} serves the requested file.
 */
@Test
public void testFileServing() throws Exception {
	final Time cacheEntryDuration = Time.milliseconds(1000L);

	final Queue<CompletableFuture<TransientBlobKey>> requestFileUploads = new ArrayDeque<>(1);

	requestFileUploads.add(CompletableFuture.completedFuture(transientBlobKey1));

	final TestTaskManagerFileHandler testTaskManagerFileHandler = createTestTaskManagerFileHandler(cacheEntryDuration, requestFileUploads, EXPECTED_TASK_MANAGER_ID);

	final File outputFile = temporaryFolder.newFile();
	final TestingChannelHandlerContext testingContext = new TestingChannelHandlerContext(outputFile);

	testTaskManagerFileHandler.respondToRequest(
		testingContext,
		HTTP_REQUEST,
		handlerRequest,
		null);

	assertThat(outputFile.length(), is(greaterThan(0L)));
	assertThat(FileUtils.readFileUtf8(outputFile), is(equalTo(fileContent1)));
}
 

@Before
public void before() throws Exception{
	File resultFile = tempFolder.newFile();
	//Delete file because the Scala API does not respect WriteMode set by the configuration
	resultFile.delete();
	resultPath = resultFile.toURI().toString();

	File verticesFile = tempFolder.newFile();
	FileUtils.writeFileUtf8(verticesFile, PageRankData.VERTICES);

	File edgesFile = tempFolder.newFile();
	FileUtils.writeFileUtf8(edgesFile, PageRankData.EDGES);

	verticesPath = verticesFile.toURI().toString();
	edgesPath = edgesFile.toURI().toString();
}
 

@Before
public void before() throws Exception {
	resultPath = tempFolder.newFile().toURI().toString();
	File verticesFile = tempFolder.newFile();
	FileUtils.writeFileUtf8(verticesFile, IncrementalSSSPData.VERTICES);

	File edgesFile = tempFolder.newFile();
	FileUtils.writeFileUtf8(edgesFile, IncrementalSSSPData.EDGES);

	File edgesInSSSPFile = tempFolder.newFile();
	FileUtils.writeFileUtf8(edgesInSSSPFile, IncrementalSSSPData.EDGES_IN_SSSP);

	verticesPath = verticesFile.toURI().toString();
	edgesPath = edgesFile.toURI().toString();
	edgesInSSSPPath = edgesInSSSPFile.toURI().toString();
}
 

/**
 * Close method, marks the I/O manager as closed
 * and removed all temporary files.
 */
public void shutdown() {
	// remove all of our temp directories
	for (File path : paths) {
		try {
			if (path != null) {
				if (path.exists()) {
					FileUtils.deleteDirectory(path);
					LOG.info("I/O manager removed spill file directory {}", path.getAbsolutePath());
				}
			}
		} catch (Throwable t) {
			LOG.error("IOManager failed to properly clean up temp file directory: " + path, t);
		}
	}
}
 

@Test
public void testComplexSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("complex-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"fn", "familyName", "additionalName", "tuples", "honorificPrefix", "url",
			"email", "tel", "sound", "org"},
		Types.STRING, Types.STRING, Types.BOOLEAN, Types.ROW(Types.BIG_DEC, Types.STRING, Types.STRING, Types.STRING),
		Types.OBJECT_ARRAY(Types.STRING), Types.STRING, Types.ROW_NAMED(new String[] {"type", "value"}, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"type", "value"}, Types.BIG_DEC, Types.STRING), Types.VOID,
		Types.ROW_NAMED(new String[] {"organizationUnit"}, Types.ROW()));

	assertEquals(expected, result);
}
 

@Override
public void close() throws IOException {
	cancelCleanupTask();

	if (shutdownRequested.compareAndSet(false, true)) {
		log.info("Shutting down BLOB cache");

		// Clean up the storage directory
		try {
			FileUtils.deleteDirectory(storageDir);
		} finally {
			// Remove shutdown hook to prevent resource leaks
			ShutdownHookUtil.removeShutdownHook(shutdownHook, getClass().getSimpleName(), log);
		}
	}
}
 

@Override
public void close() throws IOException {
	cancelCleanupTask();

	if (shutdownRequested.compareAndSet(false, true)) {
		log.info("Shutting down BLOB cache");

		// Clean up the storage directory
		try {
			FileUtils.deleteDirectory(storageDir);
		} finally {
			// Remove shutdown hook to prevent resource leaks
			ShutdownHookUtil.removeShutdownHook(shutdownHook, getClass().getSimpleName(), log);
		}
	}
}
 

@Test
public void testCleanup() {
	try {
		ByteBuffer data = ByteBuffer.allocate(157);
		data.order(ByteOrder.LITTLE_ENDIAN);

		FileUtils.writeCompletely(fileChannel, data);
		fileChannel.position(54);

		SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
		seq.open();
		seq.cleanup();

		assertFalse(fileChannel.isOpen());
		assertFalse(tempFile.exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

public void run() {

			p.destroyForcibly();

			if (pyFileDir != null) {
				File pyDir = new File(pyFileDir);
				FileUtils.deleteDirectoryQuietly(pyDir);
			}
		}
 

private static BucketState<String> readBucketStateFromTemplate(final String scenarioName, final int version) throws IOException {
	final java.nio.file.Path scenarioPath =  getResourcePath(scenarioName, version);

	// clear the scenario files first
	FileUtils.deleteDirectory(scenarioPath.toFile());

	// prepare the scenario files
	FileUtils.copy(new Path(scenarioPath.toString() + "-template"), new Path(scenarioPath.toString()), false);

	return readBucketState(scenarioName, version);
}
 

@Override
public File getFile(JobID jobId, PermanentBlobKey key) throws IOException {
	if (key.equals(permanentBlobKey)) {
		File f = temporaryFolder.newFile("cacheFile");
		FileUtils.writeFileUtf8(f, testFileContent);
		return f;
	} else {
		throw new IllegalArgumentException("This service contains only entry for " + permanentBlobKey);
	}
}
 

@BeforeClass
public static void init() throws IOException {
	final String textFileName = "test.txt";
	final String userDirHasEntryClassName = "_test_user_dir_has_entry_class";
	final String userDirHasNotEntryClassName = "_test_user_dir_has_not_entry_class";

	userDirHasEntryClass = JOB_DIRS.newFolder(userDirHasEntryClassName);
	final Path userJarPath = userDirHasEntryClass.toPath().resolve(TestJobInfo.JOB_JAR_PATH.toFile().getName());
	final Path userLibJarPath =
		userDirHasEntryClass.toPath().resolve(TestJobInfo.JOB_LIB_JAR_PATH.toFile().getName());
	userDirHasNotEntryClass = JOB_DIRS.newFolder(userDirHasNotEntryClassName);

	//create files
	Files.copy(TestJobInfo.JOB_JAR_PATH, userJarPath);
	Files.copy(TestJobInfo.JOB_LIB_JAR_PATH, userLibJarPath);
	Files.createFile(userDirHasEntryClass.toPath().resolve(textFileName));

	Files.copy(TestJobInfo.JOB_LIB_JAR_PATH, userDirHasNotEntryClass.toPath().resolve(TestJobInfo.JOB_LIB_JAR_PATH.toFile().getName()));
	Files.createFile(userDirHasNotEntryClass.toPath().resolve(textFileName));

	final Path workingDirectory = FileUtils.getCurrentWorkingDirectory();
	Arrays.asList(userJarPath, userLibJarPath)
		.stream()
		.map(path -> FileUtils.relativizePath(workingDirectory, path))
		.map(FunctionUtils.uncheckedFunction(FileUtils::toURL))
		.forEach(expectedURLs::add);
}
 

@Override
public File getFile(JobID jobId, PermanentBlobKey key) throws IOException {
	if (key.equals(permanentBlobKey)) {
		final java.nio.file.Path directory = temporaryFolder.newFolder("zipArchive").toPath();
		final java.nio.file.Path containedFile = directory.resolve("cacheFile");
		Files.copy(new ByteArrayInputStream(testFileContent.getBytes(StandardCharsets.UTF_8)), containedFile);
		Path zipPath = FileUtils.compressDirectory(new Path(directory.toString()), new Path(directory + ".zip"));
		return new File(zipPath.getPath());
	} else {
		throw new IllegalArgumentException("This service contains only entry for " + permanentBlobKey);
	}
}
 

/**
 * Creates a temporary file with the contents and returns the absolute path.
 */
private static String createTempFile(String contents) throws IOException {
	File tempFile = File.createTempFile("orders", ".csv");
	tempFile.deleteOnExit();
	FileUtils.writeFileUtf8(tempFile, contents);
	return tempFile.toURI().toString();
}
 

@Before
public void before() throws Exception {
	resultPath = tempFolder.newFile().toURI().toString();

	File edgesFile = tempFolder.newFile();
	FileUtils.writeFileUtf8(edgesFile, SingleSourceShortestPathsData.EDGES);
	edgesPath = edgesFile.toURI().toString();
}
 

@Test
public void testReadWriteCsv() throws Exception {
	// similar to CatalogTableITCase::testReadWriteCsvUsingDDL but uses HiveCatalog
	EnvironmentSettings settings = EnvironmentSettings.newInstance().useBlinkPlanner().inStreamingMode().build();
	TableEnvironment tableEnv = TableEnvironment.create(settings);
	tableEnv.getConfig().getConfiguration().setInteger(TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, 1);

	tableEnv.registerCatalog("myhive", hiveCatalog);
	tableEnv.useCatalog("myhive");

	String srcPath = this.getClass().getResource("/csv/test3.csv").getPath();

	tableEnv.executeSql("CREATE TABLE src (" +
			"price DECIMAL(10, 2),currency STRING,ts6 TIMESTAMP(6),ts AS CAST(ts6 AS TIMESTAMP(3)),WATERMARK FOR ts AS ts) " +
			String.format("WITH ('connector.type' = 'filesystem','connector.path' = 'file://%s','format.type' = 'csv')", srcPath));

	String sinkPath = new File(tempFolder.newFolder(), "csv-order-sink").toURI().toString();

	tableEnv.executeSql("CREATE TABLE sink (" +
			"window_end TIMESTAMP(3),max_ts TIMESTAMP(6),counter BIGINT,total_price DECIMAL(10, 2)) " +
			String.format("WITH ('connector.type' = 'filesystem','connector.path' = '%s','format.type' = 'csv')", sinkPath));

	TableEnvUtil.execInsertSqlAndWaitResult(tableEnv, "INSERT INTO sink " +
			"SELECT TUMBLE_END(ts, INTERVAL '5' SECOND),MAX(ts6),COUNT(*),MAX(price) FROM src " +
			"GROUP BY TUMBLE(ts, INTERVAL '5' SECOND)");

	String expected = "2019-12-12 00:00:05.0,2019-12-12 00:00:04.004001,3,50.00\n" +
			"2019-12-12 00:00:10.0,2019-12-12 00:00:06.006001,2,5.33\n";
	assertEquals(expected, FileUtils.readFileUtf8(new File(new URI(sinkPath))));
}
 

@Test
public void testCommonOptions() throws IOException, ClassNotFoundException {
	final String projectRootDir = System.getProperty("rootDir");
	final String outputDirectory = TMP.newFolder().getAbsolutePath();

	final OptionsClassLocation[] locations = new OptionsClassLocation[] {
		new OptionsClassLocation("flink-docs", TestCommonOptions.class.getPackage().getName())
	};

	ConfigOptionsDocGenerator.generateCommonSection(projectRootDir, outputDirectory, locations, "src/test/java");
	Formatter formatter = new HtmlFormatter();

	String expected =
		"<table class=\"table table-bordered\">\n" +
		"    <thead>\n" +
		"        <tr>\n" +
		"            <th class=\"text-left\" style=\"width: 20%\">Key</th>\n" +
		"            <th class=\"text-left\" style=\"width: 15%\">Default</th>\n" +
		"            <th class=\"text-left\" style=\"width: 65%\">Description</th>\n" +
		"        </tr>\n" +
		"    </thead>\n" +
		"    <tbody>\n" +
		"        <tr>\n" +
		"            <td><h5>" + TestCommonOptions.COMMON_POSITIONED_OPTION.key() + "</h5></td>\n" +
		"            <td style=\"word-wrap: break-word;\">" + TestCommonOptions.COMMON_POSITIONED_OPTION.defaultValue() + "</td>\n" +
		"            <td>" + formatter.format(TestCommonOptions.COMMON_POSITIONED_OPTION.description()) + "</td>\n" +
		"        </tr>\n" +
		"        <tr>\n" +
		"            <td><h5>" + TestCommonOptions.COMMON_OPTION.key() + "</h5></td>\n" +
		"            <td style=\"word-wrap: break-word;\">" + TestCommonOptions.COMMON_OPTION.defaultValue() + "</td>\n" +
		"            <td>" + formatter.format(TestCommonOptions.COMMON_OPTION.description()) + "</td>\n" +
		"        </tr>\n" +
		"    </tbody>\n" +
		"</table>\n";

	String output = FileUtils.readFile(Paths.get(outputDirectory, ConfigOptionsDocGenerator.COMMON_SECTION_FILE_NAME).toFile(), StandardCharsets.UTF_8.name());

	assertEquals(expected, output);
}
 

private void cleanUpPathQuietly(@Nonnull Path path) {
	try {
		FileUtils.deleteDirectory(path.toFile());
	} catch (IOException ex) {
		LOG.warn("Failed to clean up path " + path, ex);
	}
}
 

@Override
public Path call() throws IOException {
	final File file = blobService.getFile(jobID, blobKey);

	if (isDirectory) {
		Path directory = FileUtils.expandDirectory(new Path(file.getAbsolutePath()), target);
		return directory;
	} else {
		//noinspection ResultOfMethodCallIgnored
		file.setExecutable(isExecutable);
		return Path.fromLocalFile(file);
	}

}
 

/**
 * Tests that file cache entries expire.
 */
@Test
public void testFileCacheExpiration() throws Exception {
	final Time cacheEntryDuration = Time.milliseconds(5L);

	final File outputFile = runFileCachingTest(cacheEntryDuration, cacheEntryDuration);

	assertThat(outputFile.length(), is(greaterThan(0L)));
	assertThat(FileUtils.readFileUtf8(outputFile), is(equalTo(fileContent2)));
}
 

/**
 * Test that {@link FileUtils#deletePathIfEmpty(FileSystem, Path)} deletes the path if it is
 * empty. A path can only be empty if it is a directory which does not contain any
 * files/directories.
 */
@Test
public void testDeletePathIfEmpty() throws IOException {
	File file = temporaryFolder.newFile();
	File directory = temporaryFolder.newFolder();
	File directoryFile = new File(directory, UUID.randomUUID().toString());

	assertTrue(directoryFile.createNewFile());

	Path filePath = new Path(file.toURI());
	Path directoryPath = new Path(directory.toURI());
	Path directoryFilePath = new Path(directoryFile.toURI());

	FileSystem fs = FileSystem.getLocalFileSystem();

	// verify that the files have been created
	assertTrue(fs.exists(filePath));
	assertTrue(fs.exists(directoryFilePath));

	// delete the single file
	assertFalse(FileUtils.deletePathIfEmpty(fs, filePath));
	assertTrue(fs.exists(filePath));

	// try to delete the non-empty directory
	assertFalse(FileUtils.deletePathIfEmpty(fs, directoryPath));
	assertTrue(fs.exists(directoryPath));

	// delete the file contained in the directory
	assertTrue(fs.delete(directoryFilePath, false));

	// now the deletion should work
	assertTrue(FileUtils.deletePathIfEmpty(fs, directoryPath));
	assertFalse(fs.exists(directoryPath));
}
 

@Before
public void before() throws Exception {
	resultPath = tempFolder.newFile().toURI().toString();
	File verticesFile = tempFolder.newFile();
	FileUtils.writeFileUtf8(verticesFile, EuclideanGraphData.VERTICES);

	File edgesFile = tempFolder.newFile();
	FileUtils.writeFileUtf8(edgesFile, EuclideanGraphData.EDGES);

	verticesPath = verticesFile.toURI().toString();
	edgesPath = edgesFile.toURI().toString();
}
 

@Test
public void testUploadDirectoryRegeneration() throws Exception {
	OkHttpClient client = createOkHttpClientWithNoTimeouts();

	MultipartUploadResource.MultipartFileHandler fileHandler = MULTIPART_UPLOAD_RESOURCE.getFileHandler();

	FileUtils.deleteDirectory(MULTIPART_UPLOAD_RESOURCE.getUploadDirectory().toFile());

	Request fileRequest = buildFileRequest(fileHandler.getMessageHeaders().getTargetRestEndpointURL());
	try (Response response = client.newCall(fileRequest).execute()) {
		assertEquals(fileHandler.getMessageHeaders().getResponseStatusCode().code(), response.code());
	}

	verifyNoFileIsRegisteredToDeleteOnExitHook();
}
 

@Override
public void write() throws IOException {
	try {
		FileUtils.writeCompletely(this.channel.fileChannel, this.segment.wrap(0, this.segment.size()));
	}
	catch (NullPointerException npex) {
		throw new IOException("Memory segment has been released.");
	}
}
 

@Before
public void before() throws Exception {
	topSongsResultPath = tempFolder.newFile().toURI().toString();
	communitiesResultPath = tempFolder.newFile().toURI().toString();

	File tripletsFile = tempFolder.newFile();
	FileUtils.writeFileUtf8(tripletsFile, MusicProfilesData.USER_SONG_TRIPLETS);
	tripletsPath = tripletsFile.toURI().toString();

	File mismatchesFile = tempFolder.newFile();
	FileUtils.writeFileUtf8(mismatchesFile, MusicProfilesData.MISMATCHES);
	mismatchesPath = mismatchesFile.toURI().toString();
}
 

/**
 * Removes all BLOBs from local and HA store belonging to the given job ID.
 *
 * @param jobId
 * 		ID of the job this blob belongs to
 * @param cleanupBlobStoreFiles
 * 		True if the corresponding blob store files shall be cleaned up as well. Otherwise false.
 *
 * @return  <tt>true</tt> if the job directory is successfully deleted or non-existing;
 *          <tt>false</tt> otherwise
 */
public boolean cleanupJob(JobID jobId, boolean cleanupBlobStoreFiles) {
	checkNotNull(jobId);

	final File jobDir =
		new File(BlobUtils.getStorageLocationPath(storageDir.getAbsolutePath(), jobId));

	readWriteLock.writeLock().lock();

	try {
		// delete locally
		boolean deletedLocally = false;
		try {
			FileUtils.deleteDirectory(jobDir);

			// NOTE: Instead of going through blobExpiryTimes, keep lingering entries - they
			//       will be cleaned up by the timer task which tolerates non-existing files
			//       If inserted again with the same IDs (via put()), the TTL will be updated
			//       again.

			deletedLocally = true;
		} catch (IOException e) {
			LOG.warn("Failed to locally delete BLOB storage directory at " +
				jobDir.getAbsolutePath(), e);
		}

		// delete in HA blob store files
		final boolean deletedHA = !cleanupBlobStoreFiles || blobStore.deleteAll(jobId);

		return deletedLocally && deletedHA;
	} finally {
		readWriteLock.writeLock().unlock();
	}
}
 

private static void addToPythonPath(PythonEnvironment env, List<Path> pythonFiles) throws IOException {
	List<String> pythonPathList = new ArrayList<>();
	Path tmpDirPath = new Path(env.tempDirectory);

	for (Path pythonFile : pythonFiles) {
		String sourceFileName = pythonFile.getName();
		// add random UUID parent directory to avoid name conflict.
		Path targetPath = new Path(
			tmpDirPath,
			String.join(File.separator, UUID.randomUUID().toString(), sourceFileName));
		if (!pythonFile.getFileSystem().isDistributedFS()) {
			// if the path is local file, try to create symbolic link.
			new File(targetPath.getParent().toString()).mkdir();
			createSymbolicLink(
				Paths.get(new File(pythonFile.getPath()).getAbsolutePath()),
				Paths.get(targetPath.toString()));
		} else {
			try {
				FileUtils.copy(pythonFile, targetPath, true);
			} catch (Exception e) {
				LOG.error("Error occurred when copying {} to {}, skipping...", pythonFile, targetPath, e);
				continue;
			}
		}
		if (Files.isRegularFile(Paths.get(targetPath.toString()).toRealPath()) && sourceFileName.endsWith(".py")) {
			// add the parent directory of .py file itself to PYTHONPATH
			pythonPathList.add(targetPath.getParent().toString());
		} else {
			pythonPathList.add(targetPath.toString());
		}
	}

	if (env.pythonPath != null && !env.pythonPath.isEmpty()) {
		pythonPathList.add(env.pythonPath);
	}
	env.pythonPath = String.join(File.pathSeparator, pythonPathList);
}