org.sonatype.nexus.blobstore.api.BlobStoreException Java Examples

@Override
public void copy(final AmazonS3 s3, final String bucket, final String sourcePath, final String destinationPath) {
  ObjectMetadata metadataResult = s3.getObjectMetadata(bucket, sourcePath);
  long length = metadataResult.getContentLength();

  try {
    if (length < chunkSize) {
      copySinglePart(s3, bucket, sourcePath, destinationPath);
    }
    else {
      copyMultiPart(s3, bucket, sourcePath, destinationPath, length);
    }
  }
  catch(SdkClientException e) {
    throw new BlobStoreException("Error copying blob", e, null);
  }
}
 

@Override
public void upload(final AmazonS3 s3, final String bucket, final String key, final InputStream contents) {
  try (InputStream input = new BufferedInputStream(contents, chunkSize)) {
    log.debug("Starting upload to key {} in bucket {}", key, bucket);

    input.mark(chunkSize);
    ChunkReader chunkReader = new ChunkReader(input);
    Chunk chunk = chunkReader.readChunk(chunkSize).orElse(EMPTY_CHUNK);
    input.reset();

    if (chunk.dataLength < chunkSize) {
      ObjectMetadata metadata = new ObjectMetadata();
      metadata.setContentLength(chunk.dataLength);
      s3.putObject(bucket, key, new ByteArrayInputStream(chunk.data, 0, chunk.dataLength), metadata);
    }
    else {
      ChunkReader parallelReader = new ChunkReader(input);
      parallelRequests(s3, bucket, key,
          () -> (uploadId -> uploadChunks(s3, bucket, key, uploadId, parallelReader)));
    }
    log.debug("Finished upload to key {} in bucket {}", key, bucket);
  }
  catch (IOException | SdkClientException e) { // NOSONAR
    throw new BlobStoreException(format("Error uploading blob to bucket:%s key:%s", bucket, key), e, null);
  }
}
 

@Override
public void copy(final AmazonS3 s3, final String bucket, final String srcKey, final String destKey) {
  long length = s3.getObjectMetadata(bucket, srcKey).getContentLength();

  try {
    if (length < chunkSize) {
      s3.copyObject(bucket, srcKey, bucket, destKey);
    }
    else {
      final AtomicInteger offset = new AtomicInteger(1);
      parallelRequests(s3, bucket, destKey,
          () -> (uploadId -> copyParts(s3, uploadId, bucket, srcKey, destKey, length, offset)));
    }
  }
  catch (SdkClientException e) {
    throw new BlobStoreException("Error copying blob", e, null);
  }
}
 

@Test
public void anAssetBlobCanBeDeletedWhileTheSystemIsInspected() {
  AssetBlob missingAssetBlob = mockAssetBlob(mock(AssetBlob.class));
  when(asset.blob()).thenReturn(Optional.of(missingAssetBlob)); // first pass we have a missing blobRef

  FluentAsset reloadedAsset = createAsset(assetBlob);
  Blob reloadedBlob = mock(Blob.class); // second pass the blobRef is there but file does not exist
  when(reloadedBlob.getMetrics()).thenReturn(blobMetrics);
  BlobId missingBlobId = reloadedAsset.blob().get().blobRef().getBlobId();
  when(blobStore.get(missingBlobId)).thenReturn(reloadedBlob);

  mockAssetBrowse();
  mockAssetReload(reloadedAsset);

  when(reloadedBlob.getMetrics()).thenReturn(blobMetrics);
  when(reloadedBlob.getInputStream()).thenThrow(new BlobStoreException("Blob has been deleted", new BlobId("foo")));

  List<DeadBlobResult<Asset>> result = deadBlobFinder.find(repository, true);

  assertThat(result, hasSize(1));
  assertThat(result.get(0).getResultState(), is(DELETED));
}
 

@Test
public void blobIdCollisionThrowsExceptionOnRetryLimit() throws Exception {

  long size = 100L;
  HashCode sha1 = HashCode.fromString("356a192b7913b04c54574d18c28d46e6395428ab");

  Path path = util.createTempFile().toPath();

  when(fileOperations.exists(any())).thenReturn(true);

  try {
    underTest.create(path, TEST_HEADERS, size, sha1);
    fail("Expected BlobStoreException");
  }
  catch (BlobStoreException e) {
    verify(fileOperations, times(FileBlobStore.MAX_COLLISION_RETRIES + 1)).exists(any());
  }
}
 

@Override
@Guarded(by = STARTED)
public Blob create(final InputStream blobData, final Map<String, String> headers) {
  checkNotNull(blobData);

  return create(headers, destination -> {
      try (InputStream data = blobData) {
        MetricsInputStream input = new MetricsInputStream(data);
        TransferManager transferManager = new TransferManager(s3);
        transferManager.upload(getConfiguredBucket(), destination, input, new ObjectMetadata())
            .waitForCompletion();
        return input.getMetrics();
      } catch (InterruptedException e) {
        throw new BlobStoreException("error uploading blob", e, null);
      }
    });
}
 

private static InputStream packageRootAssetToInputStream(final Repository repository, final Asset packageRootAsset) {
  BlobStore blobStore = repository.facet(StorageFacet.class).blobStore();
  if (isNull(blobStore)) {
    throw new MissingAssetBlobException(packageRootAsset);
  }

  BlobRef blobRef = packageRootAsset.requireBlobRef();
  Blob blob = blobStore.get(blobRef.getBlobId());
  if (isNull(blob)) {
    throw new MissingAssetBlobException(packageRootAsset);
  }

  try {
    return blob.getInputStream();
  }
  catch (BlobStoreException ignore) { // NOSONAR
    // we want any issue with the blob store stream to be caught during the getting of the input stream as throw the
    // the same type of exception as a missing asset blob, so that we can pass the associated asset around.
    throw new MissingAssetBlobException(packageRootAsset);
  }
}
 

@Override
protected boolean doDelete(final BlobId blobId, final String reason) {
  final FileBlob blob = liveBlobs.getUnchecked(blobId);

  Lock lock = blob.lock();
  try {
    log.debug("Soft deleting blob {}", blobId);

    FileBlobAttributes blobAttributes = getFileBlobAttributes(blobId);

    if (blobAttributes == null) {
      // This could happen under some concurrent situations (two threads try to delete the same blob)
      // but it can also occur if the deleted index refers to a manually-deleted blob.
      log.warn("Attempt to mark-for-delete non-existent blob {}, hard deleting instead", blobId);
      return deleteHard(blobId);
    }
    else if (blobAttributes.isDeleted()) {
      log.debug("Attempt to delete already-deleted blob {}", blobId);
      return false;
    }

    blobAttributes.setDeleted(true);
    blobAttributes.setDeletedReason(reason);
    blobAttributes.setDeletedDateTime(new DateTime());
    blobAttributes.store();

    // record blob for hard-deletion when the next compact task runs
    deletedBlobIndex.add(blobId.toString().getBytes(UTF_8));
    blob.markStale();

    return true;
  }
  catch (Exception e) {
    throw new BlobStoreException(e, blobId);
  }
  finally {
    lock.unlock();
  }
}
 

@Nullable
@Override
@Timed
public Blob get(final BlobId blobId, final boolean includeDeleted) {
  checkNotNull(blobId);

  final S3Blob blob = liveBlobs.getUnchecked(blobId);

  if (blob.isStale()) {
    Lock lock = blob.lock();
    try {
      if (blob.isStale()) {
        S3BlobAttributes blobAttributes = new S3BlobAttributes(s3, getConfiguredBucket(), attributePath(blobId));
        boolean loaded = blobAttributes.load();
        if (!loaded) {
          log.warn("Attempt to access non-existent blob {} ({})", blobId, blobAttributes);
          return null;
        }

        if (blobAttributes.isDeleted() && !includeDeleted) {
          log.warn("Attempt to access soft-deleted blob {} attributes: {}", blobId, blobAttributes);
          return null;
        }

        blob.refresh(blobAttributes.getHeaders(), blobAttributes.getMetrics());
      }
    }
    catch (IOException e) {
      throw new BlobStoreException(e, blobId);
    }
    finally {
      lock.unlock();
    }
  }

  log.debug("Accessing blob {}", blobId);

  return blob;
}
 

@Override
protected boolean doDeleteHard(final BlobId blobId) {
  final S3Blob blob = liveBlobs.getUnchecked(blobId);
  Lock lock = blob.lock();
  try {
    log.debug("Hard deleting blob {}", blobId);

    String attributePath = attributePath(blobId);
    S3BlobAttributes blobAttributes = new S3BlobAttributes(s3, getConfiguredBucket(), attributePath);
    Long contentSize = getContentSizeForDeletion(blobAttributes);

    String blobPath = contentPath(blobId);

    boolean blobDeleted = delete(blobPath);
    delete(attributePath);

    if (blobDeleted && contentSize != null) {
      storeMetrics.recordDeletion(contentSize);
    }

    return blobDeleted;
  }
  catch (IOException e) {
    throw new BlobStoreException(e, blobId);
  }
  finally {
    lock.unlock();
    liveBlobs.invalidate(blobId);
  }
}
 

/**
 * Delete files known to be part of the S3BlobStore implementation if the content directory is empty.
 */
@Override
@Guarded(by = {NEW, STOPPED, FAILED, SHUTDOWN})
public void remove() {
  try {
    boolean contentEmpty = s3.listObjects(getConfiguredBucket(), getContentPrefix()).getObjectSummaries().isEmpty();
    if (contentEmpty) {
      S3PropertiesFile metadata = new S3PropertiesFile(s3, getConfiguredBucket(), metadataFilePath());
      metadata.remove();
      storeMetrics.remove();
      bucketManager.deleteStorageLocation(getBlobStoreConfiguration());
    }
    else {
      log.warn("Unable to delete non-empty blob store content directory in bucket {}", getConfiguredBucket());
      s3.deleteBucketLifecycleConfiguration(getConfiguredBucket());
    }
  }
  catch (AmazonS3Exception s3Exception) {
    if ("BucketNotEmpty".equals(s3Exception.getErrorCode())) {
      log.warn("Unable to delete non-empty blob store bucket {}", getConfiguredBucket());
    }
    else {
      throw new BlobStoreException(s3Exception, null);
    }
  }
  catch (IOException e) {
    throw new BlobStoreException(e, null);
  }
}
 

private Blob create(final Map<String, String> headers, final BlobIngester ingester, final BlobId blobId) {
  for (int retries = 0; retries <= MAX_COLLISION_RETRIES; retries++) {
    try {
      return tryCreate(headers, ingester, blobId);
    }
    catch (BlobCollisionException e) { // NOSONAR
      log.warn("BlobId collision: {} already exists{}", e.getBlobId(),
          retries < MAX_COLLISION_RETRIES ? ", retrying with new BlobId" : "!");
    }
  }
  throw new BlobStoreException("Cannot find free BlobId", null);
}
 

@Nullable
@Override
@Guarded(by = STARTED)
@Timed
public Blob get(final BlobId blobId, final boolean includeDeleted) {
  checkNotNull(blobId);

  final FileBlob blob = liveBlobs.getUnchecked(blobId);

  if (blob.isStale()) {
    Lock lock = blob.lock();
    try {
      if (blob.isStale()) {
        FileBlobAttributes blobAttributes = getFileBlobAttributes(blobId);
        if (blobAttributes == null) {
          return null;
        }

        if (blobAttributes.isDeleted() && !includeDeleted) {
          log.warn("Attempt to access soft-deleted blob {} attributes: {}", blobId, blobAttributes);
          return null;
        }

        blob.refresh(blobAttributes.getHeaders(), blobAttributes.getMetrics());
      }
    }
    catch (Exception e) {
      throw new BlobStoreException(e, blobId);
    }
    finally {
      lock.unlock();
    }
  }

  log.debug("Accessing blob {}", blobId);

  return blob;
}
 

public List<BlobStore> get() {
  List<BlobStore> memberList = new ArrayList<>();
  for (String name : BlobStoreGroupConfigurationHelper.memberNames(blobStoreConfiguration)) {
    BlobStore blobStore = blobStoreManager.get(name);
    if (blobStore == null) {
      throw new BlobStoreException("Blob Store '" + name + "' not found", null);
    }
    memberList.add(blobStore);
  }
  return synchronizedList(memberList);
}
 

@Override
protected boolean doDeleteHard(final BlobId blobId) {
  final FileBlob blob = liveBlobs.getUnchecked(blobId);
  Lock lock = blob.lock();
  try {
    log.debug("Hard deleting blob {}", blobId);

    Path attributePath = attributePath(blobId);
    Long contentSize = getContentSizeForDeletion(blobId);

    Path blobPath = contentPath(blobId);

    boolean blobDeleted = delete(blobPath);
    delete(attributePath);

    if (blobDeleted && contentSize != null) {
      metricsStore.recordDeletion(contentSize);
    }

    return blobDeleted;
  }
  catch (Exception e) {
    throw new BlobStoreException(e, blobId);
  }
  finally {
    lock.unlock();
    liveBlobs.invalidate(blobId);
  }
}
 

private void checkExists(final Path path, final BlobId blobId) throws IOException {
  if (!fileOperations.exists(path)) {
    // I'm not completely happy with this, since it means that blob store clients can get a blob, be satisfied
    // that it exists, and then discover that it doesn't, mid-operation
    log.warn("Can't open input stream to blob {} as file {} not found", blobId, path);
    throw new BlobStoreException("Blob has been deleted", blobId);
  }
}
 

/**
 * Delete files known to be part of the FileBlobStore implementation if the content directory is empty.
 */
@Override
@Guarded(by = {NEW, STOPPED, FAILED, SHUTDOWN})
public void remove() {
  try {
    Path blobDir = getAbsoluteBlobDir();
    if (fileOperations.deleteEmptyDirectory(contentDir)) {
      metricsStore.remove();
      fileOperations.deleteQuietly(blobDir.resolve("metadata.properties"));
      File[] files = blobDir.toFile().listFiles((dir, name) -> name.endsWith(DELETIONS_FILENAME));
      if (files != null) {
        stream(files)
            .map(File::toPath)
            .forEach(fileOperations::deleteQuietly);
      }
      else {
        log.warn("Unable to cleanup file(s) for Deletions Index");
      }
      if (!fileOperations.deleteEmptyDirectory(blobDir)) {
        log.warn("Unable to delete non-empty blob store directory {}", blobDir);
      }
    }
    else {
      log.warn("Unable to delete non-empty blob store content directory {}", contentDir);
    }
  }
  catch (Exception e) {
    throw new BlobStoreException(e, null);
  }
}
 

@Override
public void close() {
  if (deleted) {
    return;
  }
  try {
    blobStore.deleteHard(blob.getId());
    deleted = true;
  }
  catch (BlobStoreException e) {
    log.debug("Unable to delete blob {} in blob store {}", blob.getId(),
        blobStore.getBlobStoreConfiguration().getName(), e);
  }
}
 

private Blob create(final Map<String, String> headers, final CreateBlobFunction createBlobFunction) {
  BlobStore result = fillPolicy.chooseBlobStore(this, headers);
  if (result == null) {
    throw new BlobStoreException("Unable to find a member Blob Store of '" + this + "' for create", null);
  }
  Blob blob = createBlobFunction.create(result);
  locatedBlobs.put(blob.getId(), result.getBlobStoreConfiguration().getName());
  return blob;
}
 

@Override
@Guarded(by = STARTED)
public Blob copy(final BlobId blobId, final Map<String, String> headers) {
  BlobStore target = locate(blobId)
      .orElseThrow(() -> new BlobStoreException("Unable to find blob", blobId));
  Blob blob = target.copy(blobId, headers);
  locatedBlobs.put(blob.getId(), target.getBlobStoreConfiguration().getName());
  return blob;
}
 

@Override
public void upload(final AmazonS3 s3, final String bucket, final String key, final InputStream contents) {
  try (InputStream input = contents) {
    InputStream chunkOne = readChunk(input);
    if (chunkOne.available() < chunkSize) {
      uploadSinglePart(s3, bucket, key, chunkOne);
    }
    else {
      uploadMultiPart(s3, bucket, key, chunkOne, contents);
    }
  }
  catch(IOException | SdkClientException e) { // NOSONAR
    throw new BlobStoreException("Error uploading blob", e, null);
  }
}
 

@Nullable
@Override
@Guarded(by = STARTED)
@Timed
public Blob get(final BlobId blobId, final boolean includeDeleted) {
  checkNotNull(blobId);

  final GoogleCloudStorageBlob blob = liveBlobs.getUnchecked(blobId);

  if (blob.isStale()) {
    Lock lock = blob.lock();
    try {
      if (blob.isStale()) {
        GoogleCloudBlobAttributes blobAttributes = new GoogleCloudBlobAttributes(bucket, attributePath(blobId));
        boolean loaded = blobAttributes.load();
        if (!loaded) {
          log.warn("Attempt to access non-existent blob {} ({})", blobId, blobAttributes);
          return null;
        }

        if (blobAttributes.isDeleted() && !includeDeleted) {
          log.warn("Attempt to access soft-deleted blob {} ({})", blobId, blobAttributes);
          return null;
        }

        blob.refresh(blobAttributes.getHeaders(), blobAttributes.getMetrics());
      }
    }
    catch (IOException e) {
      throw new BlobStoreException(e, blobId);
    }
    finally {
      lock.unlock();
    }
  }

  log.debug("Accessing blob {}", blobId);
  return blob;
}
 

public void upload(final AmazonS3 s3, final String bucket, final String key, final InputStream contents) {
  try {
    TransferManager transferManager = TransferManagerBuilder.standard().withS3Client(s3).build();
    transferManager.upload(bucket, key, contents, new ObjectMetadata())
        .waitForCompletion();
  } catch (InterruptedException e) {
    throw new BlobStoreException("error uploading blob", e, null);
  }
}
 

/**
 * Delete files known to be part of the S3BlobStore implementation if the content directory is empty.
 */
@Override
@Guarded(by = {NEW, STOPPED, FAILED})
public void remove() {
  try {
    boolean contentEmpty = s3.listObjects(getConfiguredBucket(), CONTENT_PREFIX + "/").getObjectSummaries().isEmpty();
    if (contentEmpty) {
      S3PropertiesFile metadata = new S3PropertiesFile(s3, getConfiguredBucket(), METADATA_FILENAME);
      metadata.remove();
      storeMetrics.remove();
      s3.deleteBucket(getConfiguredBucket());
    }
    else {
      log.warn("Unable to delete non-empty blob store content directory in bucket {}", getConfiguredBucket());
    }
  }
  catch (AmazonS3Exception s3Exception) {
    if ("BucketNotEmpty".equals(s3Exception.getErrorCode())) {
      log.warn("Unable to delete non-empty blob store bucket {}", getConfiguredBucket());
    }
    else {
      throw s3Exception;
    }
  }
  catch (IOException e) {
    throw new BlobStoreException(e, null);
  }
}
 

@Override
protected boolean doDelete(final BlobId blobId, final String reason) {
  final GoogleCloudStorageBlob blob = liveBlobs.getUnchecked(blobId);

  Lock lock = blob.lock();
  try {
    log.debug("Soft deleting blob {}", blobId);

    GoogleCloudBlobAttributes blobAttributes = new GoogleCloudBlobAttributes(bucket, attributePath(blobId));

    boolean loaded = blobAttributes.load();
    if (!loaded) {
      log.warn("Attempt to mark-for-delete non-existent blob {}", blobId);
      return false;
    }
    else if (blobAttributes.isDeleted()) {
      log.debug("Attempt to delete already-deleted blob {}", blobId);
      return false;
    }

    blobAttributes.setDeleted(true);
    blobAttributes.setDeletedReason(reason);
    blobAttributes.store();

    // add the blobId to the soft-deleted index
    deletedBlobIndex.add(blobId);
    blob.markStale();

    return true;
  }
  catch (Exception e) {
    throw new BlobStoreException(e, blobId);
  }
  finally {
    lock.unlock();
  }
}
 

@Override
public void init(final BlobStoreConfiguration configuration) {
  this.blobStoreConfiguration = configuration;
  try {
    this.s3 = amazonS3Factory.create(configuration);
    if (!s3.doesBucketExist(getConfiguredBucket())) {
      s3.createBucket(getConfiguredBucket());

      if (getConfiguredExpirationInDays()>=0) {
        addBucketLifecycleConfiguration(null);
      }
    } else {
      if (getConfiguredExpirationInDays()>=0) {
        // bucket exists, we should test that the correct lifecycle config is present
        BucketLifecycleConfiguration lifecycleConfiguration = s3.getBucketLifecycleConfiguration(getConfiguredBucket());
        if (!isExpirationLifecycleConfigurationPresent(lifecycleConfiguration)) {
          addBucketLifecycleConfiguration(lifecycleConfiguration);
        }
      }
    }

    setConfiguredBucket(getConfiguredBucket());
  }
  catch (Exception e) {
    throw new BlobStoreException("Unable to initialize blob store bucket: " + getConfiguredBucket(), e, null);
  }
}
 

/**
 * @return the {@link BlobAttributes} for the blob, or null
 * @throws BlobStoreException if an {@link IOException} occurs
 */
@Override
@Guarded(by = STARTED)
public BlobAttributes getBlobAttributes(final BlobId blobId) {
  try {
    GoogleCloudBlobAttributes blobAttributes = new GoogleCloudBlobAttributes(bucket, attributePath(blobId));
    return blobAttributes.load() ? blobAttributes : null;
  }
  catch (IOException e) {
    log.error("Unable to load GoogleCloudBlobAttributes for blob id: {}", blobId, e);
    throw new BlobStoreException(e, blobId);
  }
}
 

@Override
@Guarded(by = STARTED)
public boolean deleteHard(final BlobId blobId) {
  checkNotNull(blobId);

  try {
    log.debug("Hard deleting blob {}", blobId);

    String attributePath = attributePath(blobId);
    S3BlobAttributes blobAttributes = new S3BlobAttributes(s3, getConfiguredBucket(), attributePath);
    Long contentSize = getContentSizeForDeletion(blobAttributes);

    String blobPath = contentPath(blobId);

    boolean blobDeleted = delete(blobPath);
    delete(attributePath);

    if (blobDeleted && contentSize != null) {
      storeMetrics.recordDeletion(contentSize);
    }

    return blobDeleted;
  }
  catch (IOException e) {
    throw new BlobStoreException(e, blobId);
  }
  finally {
    liveBlobs.invalidate(blobId);
  }
}
 

protected void parallelRequests(final AmazonS3 s3,
                                final String bucket,
                                final String key,
                                final Supplier<IOFunction<String, List<PartETag>>> operations)
{
  InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest(bucket, key);
  String uploadId = s3.initiateMultipartUpload(initiateRequest).getUploadId();

  CompletionService<List<PartETag>> completionService = new ExecutorCompletionService<>(executorService);
  try {
    for (int i = 0; i < parallelism; i++) {
      completionService.submit(() -> operations.get().apply(uploadId));
    }

    List<PartETag> partETags = new ArrayList<>();
    for (int i = 0; i < parallelism; i++) {
      partETags.addAll(completionService.take().get());
    }

    s3.completeMultipartUpload(new CompleteMultipartUploadRequest()
        .withBucketName(bucket)
        .withKey(key)
        .withUploadId(uploadId)
        .withPartETags(partETags));
  }
  catch (InterruptedException interrupted) {
    s3.abortMultipartUpload(new AbortMultipartUploadRequest(bucket, key, uploadId));
    Thread.currentThread().interrupt();
  }
  catch (CancellationException | ExecutionException ex) {
    s3.abortMultipartUpload(new AbortMultipartUploadRequest(bucket, key, uploadId));
    throw new BlobStoreException(
        format("Error executing parallel requests for bucket:%s key:%s with uploadId:%s", bucket, key, uploadId), ex,
        null);
  }
}
 

@Nullable
@Override
public Blob get(final BlobId blobId, final boolean includeDeleted) {
  checkNotNull(blobId);

  final S3Blob blob = liveBlobs.getUnchecked(blobId);

  if (blob.isStale()) {
    Lock lock = blob.lock();
    try {
      if (blob.isStale()) {
        S3BlobAttributes blobAttributes = new S3BlobAttributes(s3, getConfiguredBucket(), attributePath(blobId).toString());
        boolean loaded = blobAttributes.load();
        if (!loaded) {
          log.warn("Attempt to access non-existent blob {} ({})", blobId, blobAttributes);
          return null;
        }

        if (blobAttributes.isDeleted() && !includeDeleted) {
          log.warn("Attempt to access soft-deleted blob {} ({})", blobId, blobAttributes);
          return null;
        }

        blob.refresh(blobAttributes.getHeaders(), blobAttributes.getMetrics());
      }
    }
    catch (IOException e) {
      throw new BlobStoreException(e, blobId);
    }
    finally {
      lock.unlock();
    }
  }

  log.debug("Accessing blob {}", blobId);

  return blob;
}