com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion Java Examples

private static void uploadAndDeleteObjectsWithVersions() {
    System.out.println("Uploading and deleting objects with versions specified.");

    // Upload three sample objects.
    ArrayList<KeyVersion> keys = new ArrayList<KeyVersion>();
    for (int i = 0; i < 3; i++) {
        String keyName = "delete object without version ID example " + i;
        PutObjectResult putResult = S3_CLIENT.putObject(VERSIONED_BUCKET_NAME, keyName,
                "Object number " + i + " to be deleted.");
        // Gather the new object keys with version IDs.
        keys.add(new KeyVersion(keyName, putResult.getVersionId()));
    }

    // Delete the specified versions of the sample objects.
    DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(VERSIONED_BUCKET_NAME)
            .withKeys(keys)
            .withQuiet(false);

    // Verify that the object versions were successfully deleted.
    DeleteObjectsResult delObjRes = S3_CLIENT.deleteObjects(multiObjectDeleteRequest);
    int successfulDeletes = delObjRes.getDeletedObjects().size();
    System.out.println(successfulDeletes + " objects successfully deleted");
}
 

private static DeleteObjectsResult uploadAndDeleteObjectsWithoutVersions() {
    System.out.println("Uploading and deleting objects with no versions specified.");

    // Upload three sample objects.
    ArrayList<KeyVersion> keys = new ArrayList<KeyVersion>();
    for (int i = 0; i < 3; i++) {
        String keyName = "delete object with version ID example " + i;
        S3_CLIENT.putObject(VERSIONED_BUCKET_NAME, keyName, "Object number " + i + " to be deleted.");
        // Gather the new object keys without version IDs.
        keys.add(new KeyVersion(keyName));
    }

    // Delete the sample objects without specifying versions.
    DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(VERSIONED_BUCKET_NAME).withKeys(keys)
            .withQuiet(false);

    // Verify that delete markers were successfully added to the objects.
    DeleteObjectsResult delObjRes = S3_CLIENT.deleteObjects(multiObjectDeleteRequest);
    int successfulDeletes = delObjRes.getDeletedObjects().size();
    System.out.println(successfulDeletes + " objects successfully marked for deletion without versions.");
    return delObjRes;
}
 

public static void main(String[] args) throws IOException, InterruptedException {
   System.out.println("WARNING: This will indiscriminately delete every object in a given bucket. This operation cannot be undone.");
   final String bucketName = prompt("Bucket");
   if (!"yes".equals(prompt("Are you sure you want to delete all objects in bucket " + bucketName + "? (type 'yes' to continue)"))) {
      System.out.println("Aborting...");
      return;
   }
   final TruncateBucket truncater = new TruncateBucket(bucketName);
   truncater.truncate();

   ObjectListing results;
   do {
      results = s3.listObjects(bucketName);
      final List<KeyVersion> keys = results.getObjectSummaries().stream()
            .map((k) -> new KeyVersion(k.getKey()))
            .collect(toList());
      final DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucketName).withKeys(keys);
      s3.deleteObjects(deleteRequest);
   } while (results.isTruncated());

}
 

private void deleteObjects(AmazonS3 service, ObjectListing listing) throws MultiObjectDeleteException, IOException {
	do {
		if (Thread.currentThread().isInterrupted()) {
			throw new IOException(FileOperationMessages.getString("IOperationHandler.interrupted")); //$NON-NLS-1$
		}
		List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries();
		if (!objectSummaries.isEmpty()) {
			List<KeyVersion> keys = new ArrayList<KeyVersion>(objectSummaries.size());
			for (S3ObjectSummary object: objectSummaries) {
				keys.add(new KeyVersion(object.getKey()));
			}
			DeleteObjectsRequest request = new DeleteObjectsRequest(listing.getBucketName()).withKeys(keys).withQuiet(true);
			service.deleteObjects(request); // quiet
		}
		listing = service.listNextBatchOfObjects(listing);
	} while (listing.isTruncated());
}
 

@Override
public boolean delete(String path) {
  log.info("Deleting all data at location: {}", path);
  AmazonS3URI uri = toAmazonS3URI(URI.create(path));
  String bucket = uri.getBucket();

  List<KeyVersion> keysToDelete = getKeysToDelete(bucket, uri.getKey());
  log.debug("Deleting keys: {}", keysToDelete.stream().map(k -> k.getKey()).collect(Collectors.toList()));

  DeleteObjectsResult result = s3Client.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keysToDelete));
  return successfulDeletion(result, keysToDelete.size());
}
 

private List<KeyVersion> getKeysFromListing(ObjectListing listing) {
  Set<String> keys = listing.getObjectSummaries().stream().map(S3ObjectSummary::getKey).collect(Collectors.toSet());
  List<KeyVersion> keysToDelete = keys.stream().map(k -> new KeyVersion(k)).collect(Collectors.toList());

  if (!listing.isTruncated()) {
    return keysToDelete;
  }
  listing.setNextMarker(listing.getNextMarker());
  keysToDelete.addAll(getKeysFromListing(listing));

  return keysToDelete;
}
 

private static void multiObjectVersionedDeleteRemoveDeleteMarkers(DeleteObjectsResult response) {
    List<KeyVersion> keyList = new ArrayList<KeyVersion>();
    for (DeletedObject deletedObject : response.getDeletedObjects()) {
        // Note that the specified version ID is the version ID for the delete marker.
        keyList.add(new KeyVersion(deletedObject.getKey(), deletedObject.getDeleteMarkerVersionId()));
    }
    // Create a request to delete the delete markers.
    DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(VERSIONED_BUCKET_NAME).withKeys(keyList);

    // Delete the delete markers, leaving the objects intact in the bucket.
    DeleteObjectsResult delObjRes = S3_CLIENT.deleteObjects(deleteRequest);
    int successfulDeletes = delObjRes.getDeletedObjects().size();
    System.out.println(successfulDeletes + " delete markers successfully deleted");
}
 

private void delete(String bucket, Collection<String> keys) throws IOException {
  checkArgument(
      keys.size() <= MAX_DELETE_OBJECTS_PER_REQUEST,
      "only %s keys can be deleted per request, but got %s",
      MAX_DELETE_OBJECTS_PER_REQUEST,
      keys.size());
  List<KeyVersion> deleteKeyVersions =
      keys.stream().map(KeyVersion::new).collect(Collectors.toList());
  DeleteObjectsRequest request = new DeleteObjectsRequest(bucket).withKeys(deleteKeyVersions);
  try {
    amazonS3.get().deleteObjects(request);
  } catch (AmazonClientException e) {
    throw new IOException(e);
  }
}
 

@Override
public DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteObjectsRequest, AmazonS3 s3Client)
{
    LOGGER.debug("deleteObjects(): deleteObjectRequest.getBucketName() = " + deleteObjectsRequest.getBucketName() + ", deleteObjectRequest.getKeys() = " +
        deleteObjectsRequest.getKeys());

    List<DeletedObject> deletedObjects = new ArrayList<>();

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(deleteObjectsRequest.getBucketName());

    for (KeyVersion keyVersion : deleteObjectsRequest.getKeys())
    {
        String s3ObjectKey = keyVersion.getKey();
        String s3ObjectVersion = keyVersion.getVersion();
        String s3ObjectKeyVersion = s3ObjectKey + (s3ObjectVersion != null ? s3ObjectVersion : "");

        mockS3Bucket.getObjects().remove(s3ObjectKey);

        if (mockS3Bucket.getVersions().remove(s3ObjectKeyVersion) != null)
        {
            DeletedObject deletedObject = new DeletedObject();
            deletedObject.setKey(s3ObjectKey);
            deletedObject.setVersionId(s3ObjectVersion);
            deletedObjects.add(deletedObject);
        }
    }

    return new DeleteObjectsResult(deletedObjects);
}
 

/**
 * Method delete all <i>objects</i> from <i>bucketName</i> in groups by 1000
 * elements
 * 
 * @param bucketName
 * @param objects
 */
public void delete(String bucketName, List<KeyVersion> objects) {
    ReactiveSeq.fromList(objects)
               .grouped(1000)
               .forEach(l -> {
                   DeleteObjectsRequest req = new DeleteObjectsRequest(bucketName);
                   req.setKeys(l.toList());
                   client.deleteObjects(req);
               });
}
 

@Test
public void deleteObjects() {
    AmazonS3Client client = mock(AmazonS3Client.class);
    S3Utils utils = new S3Utils(
                                client, null, null, false, null);
    List<KeyVersion> keys = new ArrayList<>();
    for (int i = 0; i < 2000; i++) {
        keys.add(new KeyVersion(
                                ""));
    }

    utils.delete("", keys);

    verify(client, times(2)).deleteObjects(any());
}
 

/**
 * {@inheritDoc}
 *
 * The stub does not currently support versions so this stub only supports full deletes of the objects. Passing in a
 * list of KeyVersion specs to be deleted will completely delete those objects from the stubs.
 *
 * @see com.amazonaws.services.s3.AmazonS3#deleteObjects(com.amazonaws.services.s3.model.DeleteObjectsRequest)
 */
@Override
public DeleteObjectsResult deleteObjects(final DeleteObjectsRequest deleteObjectsRequest) {
  final List<DeleteObjectsResult.DeletedObject> deletedObjects = new ArrayList<DeletedObject>();
  final BucketInfo bucket = getBucketInfo(deleteObjectsRequest.getBucketName());
  for (final KeyVersion key : deleteObjectsRequest.getKeys()) {
    final boolean found = bucket.deleteObject(key.getKey());
    if (!deleteObjectsRequest.getQuiet() && found) {
      final DeleteObjectsResult.DeletedObject result = new DeleteObjectsResult.DeletedObject();
      result.setKey(key.getKey());
      deletedObjects.add(result);
    }
  }
  return new DeleteObjectsResult(deletedObjects);
}
 

/**
 * Delete all objects matching given prefix. This method is preferred for efficient deletion of many files
 * 
 * @param folderPrefix empty path is expected for objects in the "root" of the bucket 
 * @param searchString what pattern to be matched. This pattern will be matched against "short file name", i.e. 
 *                     the object's ID after last path separator (&quot;/&quot;).<br />
 *                     If null it means all ( string &quot;.*&quot;). 
 * @param recursive if true searches recursively for matching in nested path levels (&quot;/&quot;)
 * 
 * @return list of deleted objects
 * @throws S3OperationException in case of an error from server
 */
@PublicAtsApi
public void deleteObjects( String folderPrefix, String searchString, boolean recursive ) {

    //Alternative but not documented in S3 API: getClient().listObjectsV2(bucket, "prefix")
    ListObjectsRequest request = new ListObjectsRequest(bucketName, folderPrefix, null, recursive
                                                                                                  ? null
                                                                                                  : "/",
                                                        null);
    int totallyDeleted = 0;
    try {
        ObjectListing objectListing = s3Client.listObjects(request);
        int i = 0;
        if (searchString == null) {
            searchString = ".*"; // any string
        }
        List<KeyVersion> keysForDelete = new ArrayList<KeyVersion>(100);
        Pattern searchStringPattern = Pattern.compile(searchString);
        while (true) {
            keysForDelete.clear();
            for (Iterator<?> iterator = objectListing.getObjectSummaries().iterator(); iterator.hasNext();) {
                S3ObjectSummary objectSummary = (S3ObjectSummary) iterator.next();
                if (LOG.isTraceEnabled()) {
                    LOG.trace("listObject[" + (++i) + "]: " + objectSummary.toString());
                }

                String[] fileTokens = objectSummary.getKey().split("/");
                String s3Object = fileTokens[fileTokens.length - 1];

                Matcher matcher = searchStringPattern.matcher(s3Object);
                if (matcher.find()) {
                    keysForDelete.add(new KeyVersion(objectSummary.getKey()));
                    //allListElements.add(new S3ObjectInfo(objectSummary));
                }
            }
            if (keysForDelete.size() > 0) {
                // delete current set / batch size
                DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(bucketName).withKeys(keysForDelete)
                                                                                                    .withQuiet(false);
                DeleteObjectsResult delObjRes = s3Client.deleteObjects(multiObjectDeleteRequest);
                int currentlyDeletedCount = delObjRes.getDeletedObjects().size();
                totallyDeleted = totallyDeleted + currentlyDeletedCount;

                // verify size of deleted objects
                if (keysForDelete.size() != currentlyDeletedCount) {
                    LOG.warn("The number of actually deleted objects " + currentlyDeletedCount +
                             " does not match the expected size of " + keysForDelete.size());
                } else {
                    LOG.debug("Number of deleted S3 objects in current batch is " + currentlyDeletedCount);
                }
            }

            // more objects to retrieve (1K batch size of objects)
            if (objectListing.isTruncated()) {
                objectListing = s3Client.listNextBatchOfObjects(objectListing);
            } else {
                break;
            }
        }
        LOG.info("Successfully deleted " + totallyDeleted + " objects");
    } catch (AmazonClientException e) {
        throw new S3OperationException("Error deleting multiple objects matching pattern " + searchString 
                                       + ". Number of deleted objects is " + totallyDeleted, e);
    } 
}
 

private List<KeyVersion> getKeysToDelete(String bucket, String key) {
  return getKeysFromListing(s3Client.listObjects(bucket, key));
}