Java Code Examples for org.apache.flink.util.FileUtils#deleteDirectory()

/**
 * Close method, marks the I/O manager as closed
 * and removed all temporary files.
 */
public void shutdown() {
	// remove all of our temp directories
	for (File path : paths) {
		try {
			if (path != null) {
				if (path.exists()) {
					FileUtils.deleteDirectory(path);
					LOG.info("I/O manager removed spill file directory {}", path.getAbsolutePath());
				}
			}
		} catch (Throwable t) {
			LOG.error("IOManager failed to properly clean up temp file directory: " + path, t);
		}
	}
}
 

@Override
public void close() throws IOException {
	cancelCleanupTask();

	if (shutdownRequested.compareAndSet(false, true)) {
		log.info("Shutting down BLOB cache");

		// Clean up the storage directory
		try {
			FileUtils.deleteDirectory(storageDir);
		} finally {
			// Remove shutdown hook to prevent resource leaks
			ShutdownHookUtil.removeShutdownHook(shutdownHook, getClass().getSimpleName(), log);
		}
	}
}
 

@Override
public void run() {
	try {
		synchronized (lock) {

			Set<ExecutionAttemptID> jobRefs = jobRefHolders.get(jobID);
			if (jobRefs != null && jobRefs.isEmpty()) {
				// abort the copy
				for (Future<Path> fileFuture : entries.get(jobID).values()) {
					fileFuture.cancel(true);
				}

				//remove job specific entries in maps
				entries.remove(jobID);
				jobRefHolders.remove(jobID);

				// remove the job wide temp directories
				for (File storageDirectory : storageDirectories) {
					File tempDir = new File(storageDirectory, jobID.toString());
					FileUtils.deleteDirectory(tempDir);
				}
			}
		}
	} catch (IOException e) {
		LOG.error("Could not delete file from local file cache.", e);
	}
}
 

private void deleteUploadedFiles() {
	if (currentUploadDir != null) {
		try {
			FileUtils.deleteDirectory(currentUploadDir.toFile());
		} catch (IOException e) {
			LOG.warn("Could not cleanup uploaded files.", e);
		}
	}
}
 

@Nonnull
private SnapshotDirectory prepareLocalSnapshotDirectory(long checkpointId) throws IOException {

	if (localRecoveryConfig.isLocalRecoveryEnabled()) {
		// create a "permanent" snapshot directory for local recovery.
		LocalRecoveryDirectoryProvider directoryProvider = localRecoveryConfig.getLocalStateDirectoryProvider();
		File directory = directoryProvider.subtaskSpecificCheckpointDirectory(checkpointId);

		if (!directory.exists() && !directory.mkdirs()) {
			throw new IOException("Local state base directory for checkpoint " + checkpointId +
				" already exists: " + directory);
		}

		// introduces an extra directory because RocksDB wants a non-existing directory for native checkpoints.
		// append localDirectoryName here to solve directory collision problem when two stateful operators chained in one task.
		File rdbSnapshotDir = new File(directory, localDirectoryName);
		if (rdbSnapshotDir.exists()) {
			FileUtils.deleteDirectory(rdbSnapshotDir);
		}

		Path path = new Path(rdbSnapshotDir.toURI());
		// create a "permanent" snapshot directory because local recovery is active.
		try {
			return SnapshotDirectory.permanent(path);
		} catch (IOException ex) {
			try {
				FileUtils.deleteDirectory(directory);
			} catch (IOException delEx) {
				ex = ExceptionUtils.firstOrSuppressed(delEx, ex);
			}
			throw ex;
		}
	} else {
		// create a "temporary" snapshot directory because local recovery is inactive.
		File snapshotDir = new File(instanceBasePath, "chk-" + checkpointId);
		return SnapshotDirectory.temporary(snapshotDir);
	}
}
 

private void deleteUploadedFiles() {
	if (currentUploadDir != null) {
		try {
			FileUtils.deleteDirectory(currentUploadDir.toFile());
		} catch (IOException e) {
			LOG.warn("Could not cleanup uploaded files.", e);
		}
	}
}
 

@Test
public void testUploadDirectoryRegeneration() throws Exception {
	OkHttpClient client = createOkHttpClientWithNoTimeouts();

	MultipartUploadResource.MultipartFileHandler fileHandler = MULTIPART_UPLOAD_RESOURCE.getFileHandler();

	FileUtils.deleteDirectory(MULTIPART_UPLOAD_RESOURCE.getUploadDirectory().toFile());

	Request fileRequest = buildFileRequest(fileHandler.getMessageHeaders().getTargetRestEndpointURL());
	try (Response response = client.newCall(fileRequest).execute()) {
		assertEquals(fileHandler.getMessageHeaders().getResponseStatusCode().code(), response.code());
	}

	verifyNoFileIsRegisteredToDeleteOnExitHook();
}
 

@Override
public void run() {
	try {
		synchronized (lock) {

			Set<ExecutionAttemptID> jobRefs = jobRefHolders.get(jobID);
			if (jobRefs != null && jobRefs.isEmpty()) {
				// abort the copy
				for (Future<Path> fileFuture : entries.get(jobID).values()) {
					fileFuture.cancel(true);
				}

				//remove job specific entries in maps
				entries.remove(jobID);
				jobRefHolders.remove(jobID);

				// remove the job wide temp directories
				for (File storageDirectory : storageDirectories) {
					File tempDir = new File(storageDirectory, jobID.toString());
					FileUtils.deleteDirectory(tempDir);
				}
			}
		}
	} catch (IOException e) {
		LOG.error("Could not delete file from local file cache.", e);
	}
}
 

private static AutoCloseable getFileCloser(File path) {
	return () -> {
		try {
			FileUtils.deleteDirectory(path);
			LOG.info("FileChannelManager removed spill file directory {}", path.getAbsolutePath());
		} catch (IOException e) {
			String errorMessage = String.format("FileChannelManager failed to properly clean up temp file directory: %s", path);
			throw new IOException(errorMessage, e);
		}
	};
}
 

/**
 * Removes all BLOBs from local and HA store belonging to the given job ID.
 *
 * @param jobId
 * 		ID of the job this blob belongs to
 * @param cleanupBlobStoreFiles
 * 		True if the corresponding blob store files shall be cleaned up as well. Otherwise false.
 *
 * @return  <tt>true</tt> if the job directory is successfully deleted or non-existing;
 *          <tt>false</tt> otherwise
 */
public boolean cleanupJob(JobID jobId, boolean cleanupBlobStoreFiles) {
	checkNotNull(jobId);

	final File jobDir =
		new File(BlobUtils.getStorageLocationPath(storageDir.getAbsolutePath(), jobId));

	readWriteLock.writeLock().lock();

	try {
		// delete locally
		boolean deletedLocally = false;
		try {
			FileUtils.deleteDirectory(jobDir);

			// NOTE: Instead of going through blobExpiryTimes, keep lingering entries - they
			//       will be cleaned up by the timer task which tolerates non-existing files
			//       If inserted again with the same IDs (via put()), the TTL will be updated
			//       again.

			deletedLocally = true;
		} catch (IOException e) {
			LOG.warn("Failed to locally delete BLOB storage directory at " +
				jobDir.getAbsolutePath(), e);
		}

		// delete in HA blob store files
		final boolean deletedHA = !cleanupBlobStoreFiles || blobStore.deleteAll(jobId);

		return deletedLocally && deletedHA;
	} finally {
		readWriteLock.writeLock().unlock();
	}
}
 

private void deleteUploadedFiles() {
	if (currentUploadDir != null) {
		try {
			FileUtils.deleteDirectory(currentUploadDir.toFile());
		} catch (IOException e) {
			LOG.warn("Could not cleanup uploaded files.", e);
		}
	}
}
 

@Nonnull
private SnapshotDirectory prepareLocalSnapshotDirectory(long checkpointId) throws IOException {

	if (localRecoveryConfig.isLocalRecoveryEnabled()) {
		// create a "permanent" snapshot directory for local recovery.
		LocalRecoveryDirectoryProvider directoryProvider = localRecoveryConfig.getLocalStateDirectoryProvider();
		File directory = directoryProvider.subtaskSpecificCheckpointDirectory(checkpointId);

		if (!directory.exists() && !directory.mkdirs()) {
			throw new IOException("Local state base directory for checkpoint " + checkpointId +
				" already exists: " + directory);
		}

		// introduces an extra directory because RocksDB wants a non-existing directory for native checkpoints.
		// append localDirectoryName here to solve directory collision problem when two stateful operators chained in one task.
		File rdbSnapshotDir = new File(directory, localDirectoryName);
		if (rdbSnapshotDir.exists()) {
			FileUtils.deleteDirectory(rdbSnapshotDir);
		}

		Path path = new Path(rdbSnapshotDir.toURI());
		// create a "permanent" snapshot directory because local recovery is active.
		try {
			return SnapshotDirectory.permanent(path);
		} catch (IOException ex) {
			try {
				FileUtils.deleteDirectory(directory);
			} catch (IOException delEx) {
				ex = ExceptionUtils.firstOrSuppressed(delEx, ex);
			}
			throw ex;
		}
	} else {
		// create a "temporary" snapshot directory because local recovery is inactive.
		File snapshotDir = new File(instanceBasePath, "chk-" + checkpointId);
		return SnapshotDirectory.temporary(snapshotDir);
	}
}
 

private void cleanInstanceBasePath() {
	LOG.info("Deleting existing instance base directory {}.", instanceBasePath);

	try {
		FileUtils.deleteDirectory(instanceBasePath);
	} catch (IOException ex) {
		LOG.warn("Could not delete instance base path for RocksDB: " + instanceBasePath, ex);
	}
}
 

/**
 * Shuts down the BLOB server.
 */
@Override
public void close() throws IOException {
	cleanupTimer.cancel();

	if (shutdownRequested.compareAndSet(false, true)) {
		Exception exception = null;

		try {
			this.serverSocket.close();
		}
		catch (IOException ioe) {
			exception = ioe;
		}

		// wake the thread up, in case it is waiting on some operation
		interrupt();

		try {
			join();
		}
		catch (InterruptedException ie) {
			Thread.currentThread().interrupt();

			LOG.debug("Error while waiting for this thread to die.", ie);
		}

		synchronized (activeConnections) {
			if (!activeConnections.isEmpty()) {
				for (BlobServerConnection conn : activeConnections) {
					LOG.debug("Shutting down connection {}.", conn.getName());
					conn.close();
				}
				activeConnections.clear();
			}
		}

		// Clean up the storage directory
		try {
			FileUtils.deleteDirectory(storageDir);
		}
		catch (IOException e) {
			exception = ExceptionUtils.firstOrSuppressed(e, exception);
		}

		// Remove shutdown hook to prevent resource leaks
		ShutdownHookUtil.removeShutdownHook(shutdownHook, getClass().getSimpleName(), LOG);

		if (LOG.isInfoEnabled()) {
			LOG.info("Stopped BLOB server at {}:{}", serverSocket.getInetAddress().getHostAddress(), getPort());
		}

		ExceptionUtils.tryRethrowIOException(exception);
	}
}
 

@Override
public void close() throws IOException {
	if (uploadDirectory != null) {
		FileUtils.deleteDirectory(uploadDirectory.toFile());
	}
}
 

@After
public void clear() throws Exception {
    FileUtils.deleteDirectory(new File(MODEL_PATH));
}
 

/**
 * Cleans up BLOBs which are not referenced anymore.
 */
@Override
public void run() {
	synchronized (jobRefCounters) {
		Iterator<Map.Entry<JobID, RefCount>> entryIter = jobRefCounters.entrySet().iterator();
		final long currentTimeMillis = System.currentTimeMillis();

		while (entryIter.hasNext()) {
			Map.Entry<JobID, RefCount> entry = entryIter.next();
			RefCount ref = entry.getValue();

			if (ref.references <= 0 && ref.keepUntil > 0 && currentTimeMillis >= ref.keepUntil) {
				JobID jobId = entry.getKey();

				final File localFile =
					new File(BlobUtils.getStorageLocationPath(storageDir.getAbsolutePath(), jobId));

			/*
			 * NOTE: normally it is not required to acquire the write lock to delete the job's
			 *       storage directory since there should be no one accessing it with the ref
			 *       counter being 0 - acquire it just in case, to always be on the safe side
			 */
				readWriteLock.writeLock().lock();

				boolean success = false;
				try {
					FileUtils.deleteDirectory(localFile);
					success = true;
				} catch (Throwable t) {
					log.warn("Failed to locally delete job directory " + localFile.getAbsolutePath(), t);
				} finally {
					readWriteLock.writeLock().unlock();
				}

				// let's only remove this directory from cleanup if the cleanup was successful
				// (does not need the write lock)
				if (success) {
					entryIter.remove();
				}
			}
		}
	}
}
 

@Override
public void close() throws IOException {
	if (uploadDirectory != null) {
		FileUtils.deleteDirectory(uploadDirectory.toFile());
	}
}
 

@Override
public void discardState() throws IOException {
	ensurePath();
	FileUtils.deleteDirectory(directory.toFile());
}
 

/**
 * Calling this method will attempt delete the underlying snapshot directory recursively, if the state is
 * "ongoing". In this case, the state will be set to "deleted" as a result of this call.
 *
 * @return <code>true</code> if delete is successful, <code>false</code> otherwise.
 * @throws IOException if an exception happens during the delete.
 */
public boolean cleanup() throws IOException {
	if (state.compareAndSet(State.ONGOING, State.DELETED)) {
		FileUtils.deleteDirectory(directory.toFile());
	}
	return true;
}