com.amazonaws.services.s3.transfer.Download Java Examples

@Test
@SneakyThrows
public void getFileInputStream() {
    TransferManager transferManager = mock(TransferManager.class);
    Download download = mock(Download.class);
    when(transferManager.download(anyString(), anyString(), any())).thenReturn(download);

    ReadUtils readUtils = new ReadUtils(transferManager,System.getProperty("java.io.tmpdir"));

    InputStream fileInputStream = readUtils.getFileInputStream("bucket", "key");
    assertNotNull(fileInputStream);

    verify(transferManager, times(1)).download(anyString(), anyString(), any(File.class));
    verify(download, times(1)).waitForCompletion();

    fileInputStream.close();
}
 

/**
 * Method to download file from s3 to local file system
 * 
 * @param bucketName AWS S3 bucket name
 * @param key (example: android/apkFolder/ApkName.apk)
 * @param file (local file name)
 * @param pollingInterval (polling interval in sec for S3 download status determination)
 */
public void download(final String bucketName, final String key, final File file, long pollingInterval) {
    LOGGER.info("App will be downloaded from s3.");
    LOGGER.info(String.format("[Bucket name: %s] [Key: %s] [File: %s]", bucketName, key, file.getAbsolutePath()));
    DefaultAWSCredentialsProviderChain credentialProviderChain = new DefaultAWSCredentialsProviderChain();
    TransferManager tx = new TransferManager(
            credentialProviderChain.getCredentials());
    Download appDownload = tx.download(bucketName, key, file);
    try {
        LOGGER.info("Transfer: " + appDownload.getDescription());
        LOGGER.info("	State: " + appDownload.getState());
        LOGGER.info("	Progress: ");
        // You can poll your transfer's status to check its progress
        while (!appDownload.isDone()) {
            LOGGER.info("		transferred: " + (int) (appDownload.getProgress().getPercentTransferred() + 0.5) + "%");
            CommonUtils.pause(pollingInterval);
        }
        LOGGER.info("	State: " + appDownload.getState());
        // appDownload.waitForCompletion();
    } catch (AmazonClientException e) {
        throw new RuntimeException("File wasn't downloaded from s3. See log: ".concat(e.getMessage()));
    }
    // tx.shutdownNow();
}
 

private void downloadFile(TransferManager tx,
                          String bucketName,
                          String sourcePrefixKey,
                          String destinationFile) throws Exception{
    try {
        final File snapshotFile = new File(destinationFile);
        // Only create parent directory once, if it doesn't exist.
        final File parentDir = new File(snapshotFile.getParent());
        if (!parentDir.isDirectory()) {
            final boolean parentDirCreated = parentDir.mkdirs();
            if (!parentDirCreated) {
                LOGGER.error(
                        "Error creating parent directory for file: {}. Skipping to next",
                        destinationFile);
                return;
            }
        }
        snapshotFile.createNewFile();
        final Download download = tx.download(bucketName, sourcePrefixKey, snapshotFile);
        download.waitForCompletion();
    } catch (Exception e) {
        LOGGER.error("Error downloading the file {} : {}", destinationFile, e);
        throw new Exception(e);
    }
}
 

public static void downloadFile(String bucket_name, String key_name,
                                String file_path, boolean pause) {
    System.out.println("Downloading to file: " + file_path +
            (pause ? " (pause)" : ""));

    // snippet-start:[s3.java1.s3_xfer_mgr_download.single]
    File f = new File(file_path);
    TransferManager xfer_mgr = TransferManagerBuilder.standard().build();
    try {
        Download xfer = xfer_mgr.download(bucket_name, key_name, f);
        // loop with Transfer.isDone()
        XferMgrProgress.showTransferProgress(xfer);
        // or block with Transfer.waitForCompletion()
        XferMgrProgress.waitForCompletion(xfer);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    xfer_mgr.shutdownNow();
    // snippet-end:[s3.java1.s3_xfer_mgr_download.single]
}
 

public void startDownload(TransferManager manager, File base, String pathPrefix, S3ObjectSummary summary) throws AmazonServiceException, IOException {
    // calculate target file name
    File targetFile = FileUtils.getFile(base, summary.getKey().substring(pathPrefix.length() + 1));

    // if target file exists, only download it if newer
    if (targetFile.lastModified() < summary.getLastModified().getTime()) {
        // ensure directory above file exists
        FileUtils.forceMkdir(targetFile.getParentFile());

        // Start the download
        Download download = manager.download(summary.getBucketName(), summary.getKey(), targetFile);

        // Keep for later
        startedDownloads.add(new Memo(download, targetFile, summary.getLastModified().getTime()));
    }
}
 

@Test
public void getInputStreamSupplier()
        throws AmazonClientException, InterruptedException, IOException {
    TransferManager transferManager = mock(TransferManager.class);
    Download download = mock(Download.class);

    when(transferManager.download(anyString(), anyString(), any())).thenReturn(download);

    File file = Files.createTempFile("micro-s3", "test")
                     .toFile();
    assertTrue(file.exists());
    ReadUtils utils = new ReadUtils(transferManager, "test");

    InputStream stream = utils.getInputStream("", "", () -> file);
    assertNotNull(stream);

    assertFalse(file.exists());
}
 

private File downloadFile(String objectName) throws IOException {
	File tempFile = new File(System.getProperty("java.io.tmpdir"), objectName.replace('/', '-'));
	
	Download download = s3TransferManager.download(bucketName, objectName, tempFile);
	
	try {
	    download.waitForCompletion();
	} catch(SdkBaseException | InterruptedException e) {
	    throw new RuntimeException(e);
	}
	
	return tempFile;
}
 

@Override
public Void invoke(File localFile, VirtualChannel channel) throws IOException, InterruptedException {
	TransferManager mgr = TransferManagerBuilder.standard()
			.withS3Client(AWSClientFactory.create(this.amazonS3ClientOptions.createAmazonS3ClientBuilder(), this.envVars))
			.build();

	if (this.path == null || this.path.isEmpty() || this.path.endsWith("/")) {
		try {
			final MultipleFileDownload fileDownload = mgr.downloadDirectory(this.bucket, this.path, localFile);
			fileDownload.waitForCompletion();
			RemoteDownloader.this.taskListener.getLogger().println("Finished: " + fileDownload.getDescription());
		}
		finally {
			mgr.shutdownNow();
		}
		return null;
	} else {
		try {
			final Download download = mgr.download(this.bucket, this.path, localFile);
			download.addProgressListener((ProgressListener) progressEvent -> {
				if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) {
					RemoteDownloader.this.taskListener.getLogger().println("Finished: " + download.getDescription());
				}
			});
			download.waitForCompletion();
		}
		finally {
			mgr.shutdownNow();
		}
		return null;
	}
}
 

@Test
public void getInputStreamDefaultSupplier()
        throws AmazonClientException, InterruptedException, IOException {
    TransferManager transferManager = mock(TransferManager.class);
    Download download = mock(Download.class);

    when(transferManager.download(anyString(), anyString(), any())).thenReturn(download);

    ReadUtils utils = new ReadUtils(transferManager, System.getProperty("java.io.tmpdir"));
    InputStream stream = utils.getInputStream("", "");
    assertNotNull(stream);
    verify(download).waitForCompletion();
}
 

/**
 * {@inheritDoc}
 * <p/>
 * This implementation creates any directory that does not exist in the path to the destination directory.
 */
@Override
public MultipleFileDownload downloadDirectory(String bucketName, String keyPrefix, File destinationDirectory, TransferManager transferManager)
{
    LOGGER.debug("downloadDirectory(): bucketName = " + bucketName + ", keyPrefix = " + keyPrefix + ", destinationDirectory = " + destinationDirectory);

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);

    List<Download> downloads = new ArrayList<>();
    long totalBytes = 0;

    if (mockS3Bucket != null)
    {
        for (MockS3Object mockS3Object : mockS3Bucket.getObjects().values())
        {
            if (mockS3Object.getKey().startsWith(keyPrefix))
            {
                String filePath = destinationDirectory.getAbsolutePath() + "/" + mockS3Object.getKey();
                File file = new File(filePath);
                file.getParentFile().mkdirs(); // Create any directory in the path that does not exist.
                try (FileOutputStream fileOutputStream = new FileOutputStream(file))
                {
                    LOGGER.debug("downloadDirectory(): Writing file " + file);
                    fileOutputStream.write(mockS3Object.getData());
                    totalBytes += mockS3Object.getData().length;
                    downloads.add(new DownloadImpl(null, null, null, null, null, new GetObjectRequest(bucketName, mockS3Object.getKey()), file,
                        mockS3Object.getObjectMetadata(), false));
                }
                catch (IOException e)
                {
                    throw new RuntimeException("Error writing to file " + file, e);
                }
            }
        }
    }

    TransferProgress progress = new TransferProgress();
    progress.setTotalBytesToTransfer(totalBytes);
    progress.updateProgress(totalBytes);

    MultipleFileDownloadImpl multipleFileDownload = new MultipleFileDownloadImpl(null, progress, null, keyPrefix, bucketName, downloads);
    multipleFileDownload.setState(TransferState.Completed);
    return multipleFileDownload;
}
 

public static Download getFile(final ClientOptions clientOptions, final GetObjectRequest getObjectRequest, final File file) {
    LOGGER.debug(format("Receiving object %1$s as file %2$s from bucket %3$s using GetObjectRequest", getObjectRequest.getKey(), file.getAbsolutePath(), getObjectRequest.getBucketName()));

    return getTransferManager(clientOptions).download(getObjectRequest, file);
}
 

public static Download getFile(final ClientOptions clientOptions, final String bucketName, final String key, final File file) {
    LOGGER.debug(format("Receiving object %1$s as file %2$s from bucket %3$s", key, file.getAbsolutePath(), bucketName));

    return getTransferManager(clientOptions).download(bucketName, key, file);
}
 

@Override
public Download download(String s3BucketName, String s3Key, File file, TransferManager transferManager)
{
    return transferManager.download(s3BucketName, s3Key, file);
}
 

public Memo(Download download, File file, long timestamp) {
    this.download = download;
    this.file = file;
    this.timestamp = timestamp;
}
 

/**
 * Return the InputStream for an S3Object. This API download (via multi-part) S3 object to a local file and return a
 * InputStream to that file. The local file will be deleted when the close is called on the stream.
 *
 * @param bucketName S3 bucket name
 * @param key key for the S3Object
 *
 * @return input stream to the downloaded file
 *
 * @throws AmazonClientException
 * @throws InterruptedException
 * @throws IOException
 */
public InputStream getFileInputStream(String bucketName, String key)
    throws AmazonClientException, InterruptedException, IOException {
    File file = createTmpFile();

    Download download = transferManager.download(bucketName, key, file);
    download.waitForCompletion();
    return Files.newInputStream(file.toPath(), StandardOpenOption.DELETE_ON_CLOSE);
}
 

/**
 * Method returns InputStream from S3Object. Multi-part download is used to
 * get file. s3.tmp.dir property used to store temporary files. You can
 * specify temporary file name by using tempFileSupplier object.
 * 
 * @param bucketName
 * @param key
 *            -
 * @param tempFileSupplier
 *            - Supplier providing temporary filenames
 * @return InputStream of
 * @throws AmazonServiceException
 * @throws AmazonClientException
 * @throws InterruptedException
 * @throws IOException
 */
public InputStream getInputStream(String bucketName, String key, Supplier<File> tempFileSupplier)
        throws AmazonServiceException, AmazonClientException, InterruptedException, IOException {
    File file = tempFileSupplier.get();
    try {
        Download download = transferManager.download(bucketName, key, file);
        download.waitForCompletion();
        return new ByteArrayInputStream(FileUtils.readFileToByteArray(file));
    } finally {
        file.delete();
    }
}
 

/**
 * Schedules a new transfer to download data from Amazon S3 and save it to the specified file.
 *
 * @param s3BucketName the S3 bucket name
 * @param s3Key the S3 key
 * @param file the destination file
 * @param transferManager the transfer manager implementation to use
 *
 * @return the object that represents an asynchronous download from Amazon S3
 */
public Download download(String s3BucketName, String s3Key, File file, TransferManager transferManager);