org.sonatype.nexus.blobstore.api.BlobId Java Examples

@Before
public void setup() {
  deadBlobFinder = new DatastoreDeadBlobFinder(blobStoreManager);
  asset = createAsset();
  when(repository.getName()).thenReturn("bar");
  when(repository.facet(ContentFacet.class)).thenReturn(contentFacet);
  when(repository.optionalFacet(ContentFacet.class)).thenReturn(Optional.of(contentFacet));
  when(contentFacet.assets()).thenReturn(fluentAssets);

  BlobId blobId = new BlobId("blob-1");
  when(blobRef.getBlobId()).thenReturn(blobId);
  when(blobRef.getStore()).thenReturn("my-blobstore");
  when(blobStore.get(blobId)).thenReturn(blob);
  when(blobStoreManager.get("my-blobstore")).thenReturn(blobStore);

  when(fluentAssets.path("foo")).thenReturn(builder);
  when(builder.find()).thenReturn(Optional.of(asset));
}
 

@VisibleForTesting
Optional<BlobStore> locate(final BlobId blobId) {
  String blobStoreName = locatedBlobs.get(blobId);
  if (blobStoreName != null) {
    log.trace("{} location was cached as {}", blobId, blobStoreName);
    return Optional.ofNullable(blobStoreManager.get(blobStoreName));
  }

  BlobStore blobStore = search(blobId);
  if (blobStore != null && blobStore.isWritable()) {
    String memberName = blobStore.getBlobStoreConfiguration().getName();
    log.trace("Caching {} in member {}", blobId, memberName);
    locatedBlobs.put(blobId, memberName);
  }

  return Optional.ofNullable(blobStore);
}
 

@Nullable
@Override
@Guarded(by = STARTED)
public Blob get(final BlobId blobId, final boolean includeDeleted) {
  if (includeDeleted) {
    // check directly without using cache
    return members.get().stream()
        .map((BlobStore member) -> member.get(blobId, true))
        .filter(Objects::nonNull)
        .findAny()
        .orElse(null);
  }
  else {
    return locate(blobId)
      .map((BlobStore target) -> target.get(blobId, false))
      .orElse(null);
  }
}
 

@Nullable
@Override
public BlobAttributes getBlobAttributes(final BlobId blobId) {
  Path blobPath = attributePath(blobId);
  try {
    FileBlobAttributes blobAttributes = new FileBlobAttributes(blobPath);
    if (!blobAttributes.load()) {
      log.warn("Attempt to access non-existent blob {} ({})", blobId, attributePath(blobId));
      return null;
    }
    else {
      return blobAttributes;
    }
  }
  catch (Exception e) {
      log.error("Unable to load BlobAttributes for blob id: {}, path: {}, exception: {}",
          blobId, blobPath, e.getMessage(), log.isDebugEnabled() ? e : null);
    return null;
  }
}
 

@Test
public void getRespectsPendingDeletes() throws Exception {
  try (BlobSession<?> session = new MemoryBlobSession(blobStore)) {
    BlobId testId = session.create(blobData, headers).getId();

    assertThat(session.exists(testId), is(true));
    assertThat(session.get(testId).getId(), is(testId));
    assertThat(session.get(testId, false).getId(), is(testId));
    assertThat(session.get(testId, true).getId(), is(testId));

    session.delete(testId);

    assertThat(session.exists(testId), is(true)); // 'exists' doesn't take soft-deletes into account
    assertThat(session.get(testId), is(nullValue()));
    assertThat(session.get(testId, false), is(nullValue()));
    assertThat(session.get(testId, true).getId(), is(testId));
  }
}
 

@Test
public void itWillReturnAllBlobIdsInTheStream() {
  byte[] content = "hello".getBytes();
  Blob regularBlob = underTest.create(new ByteArrayInputStream(content), ImmutableMap.of(
      BLOB_NAME_HEADER, "example",
      CREATED_BY_HEADER, "test"));

  Blob directPathBlob = underTest.create(new ByteArrayInputStream(content), ImmutableMap.of(
      CREATED_BY_HEADER, "test",
      BLOB_NAME_HEADER, "health-check/repositoryName/file.txt",
      DIRECT_PATH_BLOB_HEADER, "true"
  ));

  List<BlobId> blobIds = underTest.getBlobIdStream().collect(Collectors.toList());
  assertThat(blobIds.size(), is(equalTo(2)));
  assertThat(blobIds, containsInAnyOrder(regularBlob.getId(), directPathBlob.getId()));
}
 

@Override
protected boolean doDeleteHard(final BlobId blobId) {
  try {
    log.debug("Hard deleting blob {}", blobId);

    boolean blobDeleted = storage.delete(getConfiguredBucketName(), contentPath(blobId));
    if (blobDeleted) {
      String attributePath = attributePath(blobId);
      BlobAttributes attributes = getBlobAttributes(blobId);
      metricsStore.recordDeletion(blobId, attributes.getMetrics().getContentSize());
      storage.delete(getConfiguredBucketName(), attributePath);
      deletedBlobIndex.remove(blobId);
    }

    return blobDeleted;
  }
  finally {
    liveBlobs.invalidate(blobId);
  }
}
 

void doCompactWithDeletedBlobIndex(@Nullable final BlobStoreUsageChecker inUseChecker) throws IOException {
  log.info("Begin deleted blobs processing");
  // only process each blob once (in-use blobs may be re-added to the index)
  ProgressLogIntervalHelper progressLogger = new ProgressLogIntervalHelper(log, 60);
  for (int counter = 0, numBlobs = deletedBlobIndex.size(); counter < numBlobs; counter++) {
    checkCancellation();
    byte[] bytes = deletedBlobIndex.peek();
    if (bytes == null) {
      return;
    }
    BlobId blobId = new BlobId(new String(bytes, UTF_8));
    FileBlob blob = liveBlobs.getIfPresent(blobId);
    if (blob == null || blob.isStale()) {
      maybeCompactBlob(inUseChecker, blobId);
      deletedBlobIndex.remove();
    }
    else {
      // still in use, so move it to end of the queue
      deletedBlobIndex.remove();
      deletedBlobIndex.add(bytes);
    }
    progressLogger.info("Elapsed time: {}, processed: {}/{}", progressLogger.getElapsed(),
        counter + 1, numBlobs);
  }
  progressLogger.flush();
}
 

@Override
@Guarded(by = STARTED)
public Blob create(final InputStream blobData, final Map<String, String> headers, @Nullable final BlobId blobId) {
  checkNotNull(blobData);
  checkNotNull(headers);
  checkIsWritable();

  checkArgument(headers.containsKey(BLOB_NAME_HEADER), "Missing header: %s", BLOB_NAME_HEADER);
  checkArgument(headers.containsKey(CREATED_BY_HEADER), "Missing header: %s", CREATED_BY_HEADER);

  long start = System.nanoTime();
  Blob blob = null;
  try {
    blob = doCreate(blobData, headers, blobId);
  }
  finally {
    long elapsed = System.nanoTime() - start;
    updateTimer("create", elapsed);
    if (blob != null) {
      performanceLogger.logCreate(blob, elapsed);
    }
  }
  return blob;
}
 

@Test
public void testExistsMethodForDirectPathBlob() {
  byte[] content = "hello".getBytes();
  final ImmutableMap<String, String> DIRECT_PATH_HEADERS = ImmutableMap.of(
      CREATED_BY_HEADER, "test",
      BLOB_NAME_HEADER, "health-check/repositoryName/file.txt",
      DIRECT_PATH_BLOB_HEADER, "true"
  );
  BlobId blobId = blobIdResolver.fromHeaders(DIRECT_PATH_HEADERS);
  //At this point the exist test should return false
  assertThat(underTest.exists(blobId), is(false));

  final Blob blob = underTest.create(new ByteArrayInputStream(content), DIRECT_PATH_HEADERS);
  assertThat(blobId.asUniqueString(), is(blob.getId().asUniqueString()));
  //Now the exist test should be true
  assertThat(underTest.exists(blob.getId()), is(true));
}
 

@Override
protected boolean doDelete(final BlobId blobId, final String reason) {
  final FileBlob blob = liveBlobs.getUnchecked(blobId);

  Lock lock = blob.lock();
  try {
    log.debug("Soft deleting blob {}", blobId);

    FileBlobAttributes blobAttributes = getFileBlobAttributes(blobId);

    if (blobAttributes == null) {
      // This could happen under some concurrent situations (two threads try to delete the same blob)
      // but it can also occur if the deleted index refers to a manually-deleted blob.
      log.warn("Attempt to mark-for-delete non-existent blob {}, hard deleting instead", blobId);
      return deleteHard(blobId);
    }
    else if (blobAttributes.isDeleted()) {
      log.debug("Attempt to delete already-deleted blob {}", blobId);
      return false;
    }

    blobAttributes.setDeleted(true);
    blobAttributes.setDeletedReason(reason);
    blobAttributes.setDeletedDateTime(new DateTime());
    blobAttributes.store();

    // record blob for hard-deletion when the next compact task runs
    deletedBlobIndex.add(blobId.toString().getBytes(UTF_8));
    blob.markStale();

    return true;
  }
  catch (Exception e) {
    throw new BlobStoreException(e, blobId);
  }
  finally {
    lock.unlock();
  }
}
 

@Nullable
@Override
public BlobAttributes getBlobAttributes(final BlobId blobId) {
  try {
    S3BlobAttributes blobAttributes = new S3BlobAttributes(s3, getConfiguredBucket(), attributePath(blobId));
    return blobAttributes.load() ? blobAttributes : null;
  }
  catch (IOException e) {
    log.error("Unable to load S3BlobAttributes for blob id: {}", blobId, e);
    return null;
  }
}
 

private void detect(final BlobStore blobStore, final Consumer<String> handler) {
  Stream<BlobId> blobIds = blobStore.getBlobIdStream();

  blobIds.forEach(id -> {
    BlobAttributes attributes = blobStore.getBlobAttributes(id);
    if (attributes != null) {
      checkIfOrphaned(handler, id, attributes);
    }
    else{
      log.warn("Skipping cleanup for blob {} because blob properties not found", id);
    }
  });
}
 

@Before
public void setup() throws IOException {
  underTest = new MavenRestoreBlobStrategy(maven2MavenPathParser, nodeAccess, repositoryManager, blobStoreManager,
      dryRunPrefix);
  properties.setProperty("@BlobStore.blob-name", "org/codehaus/plexus/plexus/3.1/plexus-3.1.pom");
  properties.setProperty("@Bucket.repo-name", "test-repo");
  properties.setProperty("size", "1000");
  properties.setProperty("@BlobStore.content-type", "application/xml");
  properties.setProperty("sha1", "b64de86ceaa4f0e4d8ccc44a26c562c6fb7fb230");

  when(repositoryManager.get("test-repo")).thenReturn(repository);
  when(repository.optionalFacet(StorageFacet.class)).thenReturn(Optional.of(storageFacet));
  when(repository.optionalFacet(MavenFacet.class)).thenReturn(Optional.of(mavenFacet));
  when(storageFacet.txSupplier()).thenReturn(() -> storageTx);

  when(storageTx.findBucket(repository)).thenReturn(bucket);

  when(blob.getId()).thenReturn(new BlobId("test"));
  when(blob.getInputStream()).thenReturn(new ByteArrayInputStream(blobBytes));

  when(maven2MavenPathParser.parsePath("org/codehaus/plexus/plexus/3.1/plexus-3.1.pom"))
      .thenReturn(mavenPath);

  when(mavenPath.getCoordinates()).thenReturn(coordinates);

  when(nodeAccess.getId()).thenReturn("node");

  when(repository.facet(MavenFacet.class)).thenReturn(mavenFacet);

  when(blobStoreManager.get("test")).thenReturn(mock(BlobStore.class));
}
 

/**
 * This is a simple existence check resulting from NEXUS-16729.  This allows clients
 * to perform a simple check and is primarily intended for use in directpath scenarios.
 */
@Override
public boolean exists(final BlobId blobId) {
  checkNotNull(blobId);
  if (!fileOperations.exists(attributePath(blobId))) {
    log.debug("Blob {} was not found during existence check", blobId);
    return false;
  }
  return true;
}
 

@Test
public void testBlobDeleted() {
  Asset asset = getMockAsset("name", TEST_HASH1);
  assets.add(asset);

  BlobAttributes blobAttributes = getMockBlobAttribues("name", "sha1", true);
  when(blobStore.getBlobAttributes(new BlobId("blob"))).thenReturn(blobAttributes);

  defaultIntegrityCheckStrategy.check(repository, blobStore, NO_CANCEL, CHECK_FAILED_HANDLER);

  verify(defaultIntegrityCheckStrategy, never()).checkAsset(any(), any());
  verify(logger).warn(BLOB_PROPERTIES_MARKED_AS_DELETED, asset.name());

  assertThat(checkFailed, is(true));
}
 

@SuppressWarnings({"unchecked", "rawtypes"})
@Before
public void setUp() {
  when(blobRefMissingStore.getStore()).thenReturn("missing");
  when(blobRefBecomesUsed.getStore()).thenReturn("default");
  when(blobRefBecomesUsed.getBlobId()).thenReturn(mock(BlobId.class));

  ContinuationArrayList<AssetBlobData> firstPage = new ContinuationArrayList<>();
  firstPage.add(newAssetBlob());
  firstPage.add(newAssetBlob());
  firstPage.add(newAssetBlob(blobRefMissingStore));
  firstPage.add(newAssetBlob());
  firstPage.add(newAssetBlob());
  when(firstPage.get(firstPage.size() - 1).nextContinuationToken()).thenReturn("NEXT");

  ContinuationArrayList<AssetBlobData> lastPage = new ContinuationArrayList<>();
  lastPage.add(newAssetBlob());
  lastPage.add(newAssetBlob(blobRefBecomesUsed));
  lastPage.add(newAssetBlob());
  when(lastPage.get(lastPage.size() - 1).nextContinuationToken()).thenReturn("EOL");

  Continuation<AssetBlobData> emptyPage = new ContinuationArrayList<>();

  when(assetBlobStore.browseUnusedAssetBlobs(BATCH_SIZE, null)).thenReturn((Continuation) firstPage);
  when(assetBlobStore.browseUnusedAssetBlobs(BATCH_SIZE, "NEXT")).thenReturn((Continuation) lastPage);
  when(assetBlobStore.browseUnusedAssetBlobs(BATCH_SIZE, "EOL")).thenReturn((Continuation) emptyPage);

  when(assetBlobStore.deleteAssetBlob(any())).thenReturn(true);

  when(formatStoreManager.assetBlobStore("content")).thenReturn(assetBlobStore);

  when(blobStoreManager.get("default")).thenReturn(blobStore);
  when(blobStore.delete(any(), any())).thenReturn(true);
}
 

/**
 * This is a simple existence check resulting from NEXUS-16729.  This allows clients
 * to perform a simple check primarily intended for use in directpath scenarios.
 */
@Override
public boolean exists(final BlobId blobId) {
  checkNotNull(blobId);
  S3BlobAttributes blobAttributes = new S3BlobAttributes(s3, getConfiguredBucket(), attributePath(blobId));
  try {
    return blobAttributes.load();
  } catch (IOException ioe) {
    log.debug("Unable to load attributes {} during existence check, exception", blobAttributes, ioe);
    return false;
  }
}
 

@Test
public void deleteBlobIfRepositoryNotFound() {
  setupOrphanedBlob();

  when(repositoryManager.get(REPOSITORY_NAME)).thenReturn(null);

  underTest.delete(repository);

  verify(blobStore).deleteHard(new BlobId(ORPHANED_BLOB_ID));
  verify(blobStore).deleteHard(new BlobId(USED_BLOB_ID));
}
 

@Override
public boolean undelete(@Nullable final BlobStoreUsageChecker inUseChecker,
                        final BlobId blobId,
                        final BlobAttributes attributes,
                        final boolean isDryRun)
{
  return members.get().stream()
      .map((BlobStore member) -> member.undelete(inUseChecker, blobId, attributes, isDryRun))
      .anyMatch((Boolean deleted) -> deleted);
}
 

/**
 * Read all the content from a blob, and compare it with the metrics on file in the blob store.
 *
 * @throws RuntimeException if there is any deviation
 */
private void readContentAndValidateMetrics(final BlobId blobId,
                                           final InputStream inputStream,
                                           final BlobMetrics metadataMetrics)
    throws NoSuchAlgorithmException, IOException
{
  final MetricsInputStream measured = new MetricsInputStream(inputStream);
  ByteStreams.copy(measured, nullOutputStream());

  checkEqual("stream length", metadataMetrics.getContentSize(), measured.getSize(), blobId);
  checkEqual("SHA1 hash", metadataMetrics.getSha1Hash(), measured.getMessageDigest(), blobId);
}
 

@Override
public Stream<BlobId> getBlobIdStream() {
  try {
    return getAttributeFilePaths()
        .map(FileAttributesLocation::new)
        .map(this::getBlobIdFromAttributeFilePath)
        .map(BlobId::new);
  }
  catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
protected boolean doDelete(final BlobId blobId, final String reason) {
  if (deleteByExpire()) {
    return expire(blobId, reason);
  }
  else {
    return doDeleteHard(blobId);
  }
}
 

@Nullable
@Override
@Guarded(by = STARTED)
public Blob get(final BlobId blobId) {
  return locate(blobId)
      .map((BlobStore target) -> target.get(blobId))
      .orElse(null);
}
 

@Override
protected Blob doCreate(final InputStream blobData, final Map<String, String> headers, @Nullable final BlobId blobId) {
  return create(headers, destination -> {
    try (InputStream data = blobData) {
      MetricsInputStream input = new MetricsInputStream(data);
      uploader.upload(s3, getConfiguredBucket(), destination, input);
      return input.getMetrics();
    }
  }, blobId);
}
 

private void deleteChangeSet(final Set<BlobId> changeSet, final String reason) {
  for (BlobId blobId : changeSet) {
    try {
      blobStore.delete(blobId, reason);
    }
    catch (Throwable e) { // NOSONAR: ignore all errors during commit/rollback
      // ...because we can't roll back any associated DB changes at this point
      log.warn("Problem deleting {}:{} while {}", storeName(), blobId, reason, e);
    }
  }
}
 

@Test
public void testUndelete_CheckInUse() throws IOException {
  when(blobStoreUsageChecker.test(eq(underTest), any(BlobId.class), anyString())).thenReturn(true);

  boolean result = underTest.undelete(blobStoreUsageChecker, new BlobId("fakeid"), attributes, false);
  assertThat(result, is(true));
  verify(attributes).setDeleted(false);
  verify(attributes).setDeletedReason(null);
  verify(attributes).store();
}
 

/**
 * Used by {@link #getDirectPathBlobIdStream(String)} to convert an s3 key to a {@link BlobId}.
 *
 * @see BlobIdLocationResolver
 */
private BlobId attributePathToDirectPathBlobId(final String s3Key) {
  checkArgument(s3Key.startsWith(DIRECT_PATH_PREFIX + "/"), "Not direct path blob path: %s", s3Key);
  checkArgument(s3Key.endsWith(BLOB_ATTRIBUTE_SUFFIX), "Not blob attribute path: %s", s3Key);
  String blobName = s3Key
      .substring(0, s3Key.length() - BLOB_ATTRIBUTE_SUFFIX.length())
      .substring(DIRECT_PATH_PREFIX.length() + 1);
  Map<String, String> headers = ImmutableMap.of(
      BLOB_NAME_HEADER, blobName,
      DIRECT_PATH_BLOB_HEADER, "true"
  );
  return blobIdLocationResolver.fromHeaders(headers);
}
 

@Override
public BlobAttributes getBlobAttributes(final BlobId blobId) {
  try {
    S3BlobAttributes blobAttributes = new S3BlobAttributes(s3, getConfiguredBucket(), attributePath(blobId));
    return blobAttributes.load() ? blobAttributes : null;
  }
  catch (IOException e) {
    log.error("Unable to load S3BlobAttributes for blob id: {}", blobId, e);
    return null;
  }
}
 

@Override
@Guarded(by = STARTED)
public Blob copy(final BlobId blobId, final Map<String, String> headers) {
  BlobStore target = locate(blobId)
      .orElseThrow(() -> new BlobStoreException("Unable to find blob", blobId));
  Blob blob = target.copy(blobId, headers);
  locatedBlobs.put(blob.getId(), target.getBlobStoreConfiguration().getName());
  return blob;
}